Merge pull request #3 from NickPresta/add-preseeding

Add preseeding

.gitignore

@@ -22,6 +22,8 @@ _testmain.go
 *.exe
 *.test
 
+*.swp
+
 httpbin/
 built/
 

Makefile

@@ -7,7 +7,7 @@ install:
 	go install -race .
 
 example: install
-	./test.sh
+	./example/test.sh
 
 test:
 	go test -race -cover -v -tags testing ./...
@@ -20,8 +20,8 @@ cover:
 
 cover-web:
 	t=`mktemp 2>/dev/null || mktemp -t 'cover'` && \
-	go test -v -tags testing -race -covermode=set -coverprofile=$$t ./... && \
-	go tool cover -html=$$t && \
+	go test -v -tags testing -race -covermode=set -coverprofile=$$t ./... ; \
+	go tool cover -html=$$t ; \
 	unlink $$t
 
 testlint:

README.md

@@ -35,12 +35,62 @@ If you have Go installed, you may `go get github.com/NickPresta/chameleon` to do
 
 To run chameleon, you can:
 
-    chameleon -data ./httpstatus -url http://httpstat.us -verbose
+    chameleon -data ./httpbin -url http://httpbin.org -verbose
 
-The directory `httpstatus` must already exist before running.
+The directory `httpbin` must already exist before running.
 
 See `chameleon -help` for more information.
 
+### Preseeding the cache
+
+If you want to configure the cache at runtime without having to depend on an external service, you may preseed the cache
+via HTTP. This is particularly useful for mocking out services which don't yet exist.
+
+To preseed a request, issue a JSON `POST` request to chameleon at the `_seed` endpoint with the following payload:
+
+Field | Description
+----- | -----------
+Method | Method is the HTTP method used to match the incoming request. Case insensitive, supports arbitrary methods
+URL | URL is the absolute or relative URL to match in requests. Only the path and querystring are used
+Body | Body is the raw content
+StatusCode | StatusCode is the [HTTP status code](http://en.wikipedia.org/wiki/List_of_HTTP_status_codes)
+Headers | Headers is a map of headers in the format of string key to string value
+
+Repeated, duplicate requests to preseed the cache will be discarded and the cache unaffected.
+
+Successful new preseed requests will return an `HTTP 201 CREATED` on success or `HTTP 500 INTERNAL SERVER ERROR`.
+Duplicate preseed requests will return an `HTTP 200 OK` on success or `HTTP 500 INTERNAL SERVER ERROR` on failure.
+
+Here is an example of preseeding the cache with a JSON response for a `GET` request for `/foobar`.
+
+```python
+import requests
+
+preseed = json.dumps({
+    'URL': '/foobar',
+    'Method': 'GET,
+    'Body': '{"key": "value"}',
+    'StatusCode': 200,
+    'Headers': {
+        'Content-Type': 'application/json',
+        'Other-Header': 'something-else',
+    }
+})
+
+response = requests.post('http://localhost:6005/_seed', data=preseed)
+if response.status_code in (200, 201):
+    # Created, or duplicate
+else:
+    # Error, print it out
+    print(response.content)
+
+# Continue tests as normal
+# Making requests to `/foobar` will return `{"key": "value"}`
+# without hitting the proxied service
+```
+
+Check out the [example](./example) directory to see preseeding in action.
+
 ### How chameleon caches responses
 
 chameleon makes a hash for a given request URI and method and uses that to cache content. What that means:
@@ -109,7 +159,7 @@ Field | Description
 ----- | -----------
 BodyBase64 | Body is the request's body, base64 encoded
 ContentLength | ContentLength records the length of the associated content after being base64 decoded
-Headers | Headers is a map of request lines to value lists. HTTP defines that header names are case-insensitive. Header names have been canonicalized, making the first character and any characters following a hyphen uppercase and the rest lowercase.
+Headers | Headers is a map of request lines to value lists. HTTP defines that header names are case-insensitive. Header names have been canonicalized, making the first character and any characters following a hyphen uppercase and the rest lowercase
 Method | Method specifies the HTTP method (`GET`, `POST`, `PUT`, etc.)
 URL | URL is an object containing `Host`, the HTTP Host in the form of 'host' or 'host:port', `Path`, the request path including trailing slash, `RawQuery`, encoded query string values without '?', and `Scheme`, the URL scheme 'http', 'https'
 
@@ -122,6 +172,8 @@ general troubleshooting.
 
 [Nick Presta](http://nickpresta.ca) ([@NickPresta](https://twitter.com/NickPresta))
 
+Thanks to [@mdibernardo](https://twitter.com/mdibernardo) for the inspiration.
+
 ## License
 
 Please see [LICENSE](./LICENSE)

cache.go

@@ -124,34 +124,41 @@ func (c DiskCacher) Put(key string, resp *httptest.ResponseRecorder) *CachedResp
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 
-	specs := c.loadSpecs()
+	skipDisk := resp.Header().Get("_chameleon-seeded-skip-disk") != ""
+	if skipDisk {
+		resp.Header().Del("_chameleon-seeded-skip-disk")
+	}
 
 	specHeaders := make(map[string]string)
 	for k, v := range resp.Header() {
 		specHeaders[k] = strings.Join(v, ", ")
 	}
 
-	newSpec := Spec{
-		Key: key,
-		SpecResponse: SpecResponse{
-			StatusCode:  resp.Code,
-			ContentFile: key,
-			Headers:     specHeaders,
-		},
-	}
+	if !skipDisk {
+		specs := c.loadSpecs()
 
-	specs = append(specs, newSpec)
+		newSpec := Spec{
+			Key: key,
+			SpecResponse: SpecResponse{
+				StatusCode:  resp.Code,
+				ContentFile: key,
+				Headers:     specHeaders,
+			},
+		}
 
-	contentFilePath := path.Join(c.dataDir, key)
-	err := c.FileSystem.WriteFile(contentFilePath, resp.Body.Bytes())
-	if err != nil {
-		panic(err)
-	}
+		specs = append(specs, newSpec)
 
-	specBytes, err := json.MarshalIndent(specs, "", "    ")
-	err = c.FileSystem.WriteFile(c.specPath, specBytes)
-	if err != nil {
-		panic(err)
+		contentFilePath := path.Join(c.dataDir, key)
+		err := c.FileSystem.WriteFile(contentFilePath, resp.Body.Bytes())
+		if err != nil {
+			panic(err)
+		}
+
+		specBytes, err := json.MarshalIndent(specs, "", "    ")
+		err = c.FileSystem.WriteFile(c.specPath, specBytes)
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	c.cache[key] = &CachedResponse{

cache_test.go

@@ -93,3 +93,17 @@ func TestDiskCacherSeedCacheNoSpecs(t *testing.T) {
 		t.Errorf("Got: `%v`; Expected: `0`", len(cacher.cache))
 	}
 }
+
+func TestDiskCacherPutSkipDiskSeeded(t *testing.T) {
+	cacher := NewDiskCacher("")
+	cacher.FileSystem = mockFileSystem{}
+	cacher.SeedCache()
+
+	recorder := httptest.NewRecorder()
+	recorder.Header().Set("_chameleon-seeded-skip-disk", "true")
+	response := cacher.Put("new_key", recorder)
+
+	if _, ok := response.Headers["_chameleon-seeded-skip-disk"]; ok {
+		t.Errorf("Unexpected header `_chameleon-seeded-skip-disk`")
+	}
+}

example/README.md

@@ -12,7 +12,7 @@ grabs the "message" for that status code (`OK`, `I'M A TEAPOT`, etc) and returns
 
 To run this application (you need Python 2.x):
 
-    $ TEST_PORT=10005 python app.py
+    $ TEST_APP_PORT=10005 python app.py
 
 Then use cURL, your browser, etc, and issue an HTTP GET request to `localhost:10005/418`. You should see `I'M A TEAPOT`
 as the response body.
@@ -21,26 +21,26 @@ as the response body.
 
 There are some accompanying user tests (E2E, API, what ever you call them) in the file `tests.py`. Run it like so:
 
-    $ TEST_PORT=10005 python tests.py
+    $ TEST_APP_PORT=10005 python tests.py
 
 You should see a bunch of unit tests pass that look like this (note the time it takes):
 
-    $ TEST_PORT=10005 python tests.py
-    ........
+    $ TEST_APP_PORT=10005 python tests.py
+    .........
     ----------------------------------------------------------------------
-    Ran 8 tests in 1.970s
+    Ran 9 tests in 3.9s
 
     OK
 
 You could imagine tests that check JSON error payloads conform to a certain structure, that response headers are present,
 and a whole list of other things you care about in an end-to-end test scenario.
 
-## Applicability
+## Problems
 
-Imagine you are writing an app that depended on an external service to do its job.
+Imagine you are writing an app that depended on an external service to do its job. Perhaps this is the Twitter Search API, or something equally restrictive in the number of times you're allowed to interact with it.
 
 What would you do if your external service was rate limiting you? How about only allowing access from specific
-IP addresses? What if the external service was slow?
+IP addresses? What if the external service was slow and unreliable?
 
 You could proxy and cache the backend service and allow your E2E tests to behave normally and with real, valid data.
 
@@ -55,15 +55,15 @@ This assumes you're running chameleon from this `example` directory.
 
 1. Instruct our application to use chameleon to make requests. We set the `TEST_SERVICE_URL` to chameleon:
 
-        $ TEST_PORT=10005 TEST_SERVICE_URL=http://localhost:6005/ python app.py
+        $ TEST_APP_PORT=10005 TEST_SERVICE_URL=http://localhost:6005/ python app.py
 
 1. Run our tests again:
 
 
-        $ TEST_PORT=10005 python tests.py
-        ........
+        $ TEST_APP_PORT=10005 TEST_CHAMELEON_PORT=6005 python tests.py
+        .........
         ----------------------------------------------------------------------
-        Ran 8 tests in 1.728s
+        Ran 9 tests in 3.398s
 
         OK
 
@@ -80,14 +80,14 @@ it had a cache for the `200` code so it returned the cached version.
 
 If we run our tests again, we see:
 
-    $ TEST_PORT=10005 python tests.py
-    ........
+    $ TEST_APP_PORT=10005 TEST_CHAMELEON_PORT=6005 python tests.py
+    .........
     ----------------------------------------------------------------------
-    Ran 8 tests in 0.015s
+    Ran 9 tests in 0.415s
 
     OK
 
-In all four cases, chameleon returned the responses from disk. This resulted in a much faster test run,
+In all nine cases, chameleon already has the responses in memory. This resulted in a much faster test run,
 and if our backend service started to throttle us, or we wanted to run these tests from somewhere that couldn't
 reach httpbin, we still could.
 

example/app.py

@@ -6,7 +6,7 @@
 import urlparse
 
 SERVICE_URL = os.getenv('TEST_SERVICE_URL', 'https://httpbin.org/')
-PORT = int(os.getenv('TEST_PORT', 9001))
+TEST_APP_PORT = int(os.getenv('TEST_APP_PORT', 9001))
 
 STATUS_SERVICE_URL = urlparse.urljoin(SERVICE_URL, '/status/')
 POST_SERVICE_URL = urlparse.urljoin(SERVICE_URL, '/post')
@@ -75,10 +75,21 @@ def do_HASHED(self):
         # Custom method that hashes a post with body
         self._do_patch_post_put(POST_SERVICE_URL, 'POST', {'chameleon-hash-body': 'true'})
 
+    def do_SEEDED(self):
+        url = urlparse.urljoin(SERVICE_URL, self.path[1:])
+        try:
+            resp = urllib2.urlopen(url)
+        except urllib2.HTTPError as exc:
+            resp = exc
+        self.send_response(resp.getcode())
+        self.send_header('Content-type', resp.headers['content-type'])
+        self.end_headers()
+        self.wfile.write(resp.read())
+
 
 def main():
-    print('Serving on port {}'.format(PORT))
-    server = BaseHTTPServer.HTTPServer(('localhost', PORT), MyHandler)
+    print('Serving on port {}'.format(TEST_APP_PORT))
+    server = BaseHTTPServer.HTTPServer(('localhost', TEST_APP_PORT), MyHandler)
     server.serve_forever()
 
 

example/test.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+export TEST_APP_PORT=10010
+export TEST_CHAMELEON_PORT=6010
+
+trap 'kill $(jobs -p) > /dev/null 2>&1' EXIT  # Cleanup our servers on exit
+
+cd $(dirname $0)
+
+chameleon -data ./testing_data -url https://httpbin.org/ -hasher="python ./hasher.py" \
+    -host localhost:$TEST_CHAMELEON_PORT -verbose &
+TEST_SERVICE_URL=http://localhost:$TEST_CHAMELEON_PORT/ python app.py > /dev/null 2>&1 &
+
+sleep 3  # Let the servers spin up
+
+python tests.py > results.txt 2>&1
+TEST_RESULT=$?
+
+cat results.txt
+rm -f results.txt
+
+exit $TEST_RESULT

testing_data/spec.json → example/testing_data/spec.json

@@ -127,4 +127,4 @@
         },
         "key": "9835adf25e3ecc09431cdf3079bb822a"
     }
-]
+]