I think something like this might work:

var hasWebP = false;
(function() {
  var img = new Image();
  img.onload = function() {
    hasWebP = !!(img.height > 0 && img.width > 0);
  };
  img.onerror = function() {
    hasWebP = false;
  };
  img.src = 'http://www.gstatic.com/webp/gallery/1.webp';
})();

In Firefox and IE, the "onload" handler just won't be called at all if the image can't be understood, and the "onerror" is called instead.

You didn't mention jQuery, but as an example of how to deal with the asynchronous nature of that check you could return a jQuery "Deferred" object:

function hasWebP() {
  var rv = $.Deferred();
  var img = new Image();
  img.onload = function() { rv.resolve(); };
  img.onerror = function() { rv.reject(); };
  img.src = 'http://www.gstatic.com/webp/gallery/1.webp';
  return rv.promise();
}

Then you could write:

hasWebP().then(function() {
  // ... code to take advantage of WebP ...
}, function() {
  // ... code to deal with the lack of WebP ...
});

Here is a jsfiddle example.

---

A more advanced checker: http://jsfiddle.net/JMzj2/29/. This one loads images from a data URL and checks whether it loads successfully. Since WebP now also supports lossless images, you could check whether the current browser supports just lossy WebP or also lossless WebP. (Note: This implicitly also checks for data URL support.)

var hasWebP = (function() {
    // some small (2x1 px) test images for each feature
    var images = {
        basic: "data:image/webp;base64,UklGRjIAAABXRUJQVlA4ICYAAACyAgCdASoCAAEALmk0mk0iIiIiIgBoSygABc6zbAAA/v56QAAAAA==",
        lossless: "data:image/webp;base64,UklGRh4AAABXRUJQVlA4TBEAAAAvAQAAAAfQ//73v/+BiOh/AAA="
    };

    return function(feature) {
        var deferred = $.Deferred();

        $("<img>").on("load", function() {
            // the images should have these dimensions
            if(this.width === 2 && this.height === 1) {
                deferred.resolve();
            } else {
                deferred.reject();
            }
        }).on("error", function() {
            deferred.reject();
        }).attr("src", images[feature || "basic"]);

        return deferred.promise();
    }
})();

var add = function(msg) {
    $("<p>").text(msg).appendTo("#x");
};

hasWebP().then(function() {
    add("Basic WebP available");
}, function() {
    add("Basic WebP *not* available");
});

hasWebP("lossless").then(function() {
    add("Lossless WebP available");
}, function() {
    add("Lossless WebP *not* available");
});