First attempt at some new caching; including in-memory, on-disk, and on S3

.idea/jsLibraryMappings.xml

@@ -4,4 +4,4 @@
     <file url="file://$PROJECT_DIR$" libraries="{compiler-explorer/node_modules}" />
     <includedPredefinedLibrary name="Node.js Core" />
   </component>
-</project>
+</project>

lib/cache/base.js

@@ -0,0 +1,93 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const crypto = require('crypto'),
+    logger = require('../logger').logger;
+
+const HashVersion = 'Compiler Explorer Cache Version 1';
+const ReportEveryMs = 5 * 60 * 1000;
+
+class BaseCache {
+    constructor(name) {
+        this.name = name;
+        this.gets = 0;
+        this.hits = 0;
+        this.puts = 0;
+        this.reporter = setInterval(() => this.report(), ReportEveryMs);
+    }
+
+    dispose() {
+        clearInterval(this.reporter);
+    }
+
+    stats() {
+        return {hits: this.hits, puts: this.puts, gets: this.gets};
+    }
+
+    statString() {
+        const pc = this.gets ? (100 * this.hits) / this.gets : 0;
+        const misses = this.gets - this.hits;
+        return `${this.puts} puts; ${this.gets} gets, ${this.hits} hits, ${misses} misses (${pc.toFixed(2)}%)`;
+    }
+
+    report() {
+        logger.info(`${this.name}: cache stats: ${this.statString()}`);
+    }
+
+    static hash(object) {
+        return crypto.createHmac('sha256', HashVersion)
+            .update(JSON.stringify(object))
+            .digest('hex');
+    }
+
+    get(key) {
+        this.gets++;
+        return this.getInternal(key)
+            .then(result => {
+                if (result.hit) {
+                    this.hits++;
+                }
+                return result;
+            });
+    }
+
+    put(key, value, creator) {
+        if (!(value instanceof Buffer))
+            value = new Buffer(value);
+        this.puts++;
+        return this.putInternal(key, value, creator);
+    }
+
+    // eslint-disable-next-line no-unused-vars
+    getInternal(key) {
+        return Promise.reject("should be implemented in subclass");
+    }
+
+    // eslint-disable-next-line no-unused-vars
+    putInternal(key, value, creator) {
+        return Promise.reject("should be implemented in subclass");
+    }
+}
+
+module.exports = BaseCache;

lib/cache/in-memory.js

@@ -0,0 +1,59 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const LRU = require('lru-cache'),
+    BaseCache = require('./base.js');
+
+class InMemoryCache extends BaseCache {
+    constructor(cacheMb) {
+        super(`InMemoryCache(${cacheMb}Mb)`);
+        this.cache = LRU({
+            max: cacheMb * 1024 * 1024,
+            length: n => {
+                if (n instanceof Buffer)
+                    return n.length;
+                return JSON.stringify(n).length;
+            }
+        });
+    }
+
+    statString() {
+        return `${super.statString()}, LRU has ${this.cache.itemCount} item(s) totalling ${this.cache.length} bytes`;
+    }
+
+    getInternal(key) {
+        const cached = this.cache.get(key);
+        return Promise.resolve({
+            hit: !!cached,
+            data: cached
+        });
+    }
+
+    putInternal(key, value/*, creator*/) {
+        this.cache.set(key, value);
+        return Promise.resolve();
+    }
+}
+
+module.exports = InMemoryCache;

lib/cache/multi.js

@@ -0,0 +1,58 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const BaseCache = require('./base.js');
+
+// A write-through multiple cache.
+// Writes get pushed to all caches, but reads are serviced from the first cache that returns
+// a hit.
+class MultiCache extends BaseCache {
+    constructor(...upstream) {
+        super("Multi");
+        this.upstream = upstream;
+    }
+
+    statString() {
+        return `${super.statString()}. ${this.upstream.map(c => `${c.name}: ${c.statString()}`).join(". ")}`;
+    }
+
+    getInternal(key) {
+        let promiseChain = Promise.resolve({hit: false});
+        for (let cache of this.upstream) {
+            promiseChain = promiseChain.then(upstream => {
+                if (upstream.hit) return upstream;
+                return cache.get(key);
+            });
+        }
+        return promiseChain;
+    }
+
+    putInternal(object, value, creator) {
+        return Promise.all(this.upstream.map(cache => {
+            return cache.put(object, value, creator);
+        }));
+    }
+}
+
+module.exports = MultiCache;

lib/cache/on-disk.js

@@ -0,0 +1,104 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const LRU = require('lru-cache'),
+    fs = require('fs'),
+    path = require('path'),
+    mkdirp = require('mkdirp'),
+    BaseCache = require('./base.js'),
+    denodeify = require('denodeify');
+
+// With thanks to https://gist.github.com/kethinov/6658166
+function getAllFiles(root, dir) {
+    dir = dir || root;
+    return fs.readdirSync(dir).reduce((files, file) => {
+        const fullPath = path.join(dir, file);
+        const name = path.relative(root, fullPath);
+        const isDirectory = fs.statSync(fullPath).isDirectory();
+        return isDirectory ? [...files, ...getAllFiles(root, fullPath)] : [...files, {name, fullPath}];
+    }, []);
+}
+
+const readFile = denodeify(fs.readFile);
+const writeFile = denodeify(fs.writeFile);
+
+class OnDiskCache extends BaseCache {
+    constructor(path, cacheMb) {
+        super(`OnDiskCache(${path}, ${cacheMb}mb)`);
+        this.path = path;
+        this.cache = LRU({
+            max: cacheMb * 1024 * 1024,
+            length: n => n.size,
+            dispose: (key, n) => {
+                fs.unlink(n.path, () => {});
+            }
+        });
+        mkdirp.sync(path);
+        const info = getAllFiles(path).map(({name, fullPath}) => {
+            const stat = fs.statSync(fullPath);
+            return {
+                key: name,
+                sort: stat.ctimeMs,
+                data: {
+                    path: fullPath,
+                    size: stat.size
+                }
+            };
+        });
+        // Sort oldest first
+        info.sort((x, y) => {
+            return x.sort - y.sort;
+        });
+        for (let i of info) {
+            this.cache.set(i.key, i.data);
+        }
+    }
+
+    statString() {
+        return `${super.statString()}, LRU has ${this.cache.itemCount} item(s) ` +
+            `totalling ${this.cache.length} bytes on disk`;
+    }
+
+    getInternal(key) {
+        const cached = this.cache.get(key);
+        if (!cached) return Promise.resolve({hit: false});
+        return readFile(cached.path)
+            .then((data) => {
+                return {hit: true, data: data};
+            });
+    }
+
+    putInternal(key, value) {
+        const info = {
+            path: path.join(this.path, key),
+            size: value.length
+        };
+        return writeFile(info.path, value)
+            .then(() => {
+                this.cache.set(key, info);
+            });
+    }
+}
+
+module.exports = OnDiskCache;

lib/cache/s3.js

@@ -0,0 +1,63 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const BaseCache = require('./base.js');
+const AWS = require('aws-sdk');
+
+class S3Cache extends BaseCache {
+    constructor(bucket, path, region) {
+        super(`S3Cache(s3://${bucket}/${path} in ${region})`);
+        this.bucket = bucket;
+        this.path = path;
+        this.s3 = new AWS.S3({region});
+    }
+
+    statString() {
+        return `${super.statString()}, LRU has ${this.cache.itemCount} item(s) ` +
+            `totalling ${this.cache.length} bytes on disk`;
+    }
+
+    getInternal(key) {
+        return this.s3.getObject({Bucket: this.bucket, Key: `${this.path}/${key}`})
+            .promise()
+            .then((result) => {
+                return {hit: true, data: result.Body};
+            })
+            .catch((x) => {
+                if (x.code === 'NoSuchKey') return {hit: false};
+                throw x;
+            });
+    }
+
+    putInternal(key, value, creator) {
+        return this.s3.putObject({
+            Bucket: this.bucket, Key: `${this.path}/${key}`, Body: value,
+            StorageClass: "REDUCED_REDUNDANCY",
+            Metadata: {CreatedBy: creator}
+        })
+            .promise();
+    }
+}
+
+module.exports = S3Cache;

package.json

@@ -36,6 +36,7 @@
     "jquery": "^3.3.1",
     "lru-cache": "^4.1.2",
     "lz-string": "^1.4.4",
+    "mkdirp": "^0.5.1",
     "monaco-editor": "0.10.1",
     "morgan": "^1.9.0",
     "nopt": "3.0.x",

test/cache-tests.js

@@ -0,0 +1,208 @@
+// Copyright (c) 2018, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ,
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+const chai = require('chai');
+const chaiAsPromised = require("chai-as-promised");
+const InMemoryCache = require('../lib/cache/in-memory');
+const MultiCache = require('../lib/cache/multi');
+const OnDiskCache = require('../lib/cache/on-disk');
+const S3Cache = require('../lib/cache/s3');
+const temp = require('temp');
+const fs = require('fs');
+const path = require('path');
+const AWS = require('aws-sdk-mock');
+
+chai.use(chaiAsPromised);
+chai.should();
+
+function newTempDir() {
+    temp.track(true);
+    return temp.mkdirSync({prefix: 'compiler-explorer-cache-tests', dir: process.env.tmpDir});
+}
+
+function basicTests(factory) {
+    it('should start empty', () => {
+        const cache = factory();
+        cache.stats().should.eql({hits: 0, puts: 0, gets: 0});
+        return cache.get('not a key', 'subsystem').should.eventually.contain({hit: false})
+            .then((x) => {
+                cache.stats().should.eql({hits: 0, puts: 0, gets: 1});
+                return x;
+            });
+    });
+
+    it('should store and retrieve strings', () => {
+        const cache = factory();
+        return cache.put('a key', 'a value', 'bob')
+            .then(() => {
+                cache.stats().should.eql({hits: 0, puts: 1, gets: 0});
+                return cache.get('a key').should.eventually.eql({
+                    hit: true,
+                    data: new Buffer('a value')
+                });
+            }).then(x => {
+                cache.stats().should.eql({hits: 1, puts: 1, gets: 1});
+                return x;
+            });
+    });
+
+    it('should store and retrieve binary buffers', () => {
+        const cache = factory();
+        const buffer = new Buffer(2 * 1024 * 1024);
+        buffer.fill('@');
+        return cache.put('a key', buffer, 'bob')
+            .then(() => {
+                cache.stats().should.eql({hits: 0, puts: 1, gets: 0});
+                return cache.get('a key').should.eventually.eql({
+                    hit: true,
+                    data: buffer
+                });
+            }).then(x => {
+                cache.stats().should.eql({hits: 1, puts: 1, gets: 1});
+                return x;
+            });
+    });
+}
+
+describe('In-memory caches', () => {
+    basicTests(() => new InMemoryCache(10));
+    it('should give extra stats', () => {
+        const cache = new InMemoryCache(1);
+        cache.statString().should.equal(
+            '0 puts; 0 gets, 0 hits, 0 misses (0.00%), LRU has 0 item(s) totalling 0 bytes');
+    });
+
+    it('should evict old objects', () => {
+        const cache = new InMemoryCache(1);
+        return cache.put('a key', 'a value', 'bob')
+            .then(() => {
+                const promises = [];
+                const oneK = "".padEnd(1024);
+                for (let i = 0; i < 1024; i++) {
+                    promises.push(cache.put(`key${i}`, oneK));
+                }
+                return Promise.all(promises);
+            })
+            .then(() => {
+                return cache.get('a key').should.eventually.contain({hit: false});
+            });
+    });
+});
+
+describe('Multi caches', () => {
+    basicTests(() => new MultiCache(new InMemoryCache(10), new InMemoryCache(20), new InMemoryCache(30)));
+
+    it('should write through', () => {
+        const subCache1 = new InMemoryCache(1);
+        const subCache2 = new InMemoryCache(1);
+        const cache = new MultiCache(subCache1, subCache2);
+        return cache.put('a key', 'a value', 'bob')
+            .then(() => {
+                return Promise.all([
+                    cache.get('a key').should.eventually.eql({hit: true, data: new Buffer('a value')}),
+                    subCache1.get('a key').should.eventually.eql({hit: true, data: new Buffer('a value')}),
+                    subCache2.get('a key').should.eventually.eql({hit: true, data: new Buffer('a value')})
+                ]);
+            });
+    });
+
+    it('services from the first cache hit', () => {
+        const subCache1 = new InMemoryCache(1);
+        const subCache2 = new InMemoryCache(1);
+        // Set up caches with deliberately skew values for the same key.
+        subCache1.put('a key', 'cache1');
+        subCache2.put('a key', 'cache2');
+        const cache = new MultiCache(subCache1, subCache2);
+        return cache.get('a key').should.eventually.eql({hit: true, data: new Buffer('cache1')})
+            .then((x) => {
+                subCache1.hits.should.equal(1);
+                subCache1.gets.should.equal(1);
+                subCache2.hits.should.equal(0);
+                subCache2.gets.should.equal(0);
+                return x;
+            }).then(() => {
+                Promise.all([
+                    subCache1.get('a key').should.eventually.eql({hit: true, data: new Buffer('cache1')}),
+                    subCache2.get('a key').should.eventually.eql({hit: true, data: new Buffer('cache2')})]
+                );
+            });
+    });
+});
+
+describe('On disk caches', () => {
+    basicTests(() => new OnDiskCache(newTempDir(), 10));
+    it('should evict old objects', () => {
+        const tempDir = newTempDir();
+        const cache = new OnDiskCache(tempDir, 1);
+        return cache.put('a key', 'a value', 'bob')
+            .then(() => {
+                const promises = [];
+                const oneK = "".padEnd(1024);
+                for (let i = 0; i < 1024; i++) {
+                    promises.push(cache.put(`key${i}`, oneK));
+                }
+                return Promise.all(promises);
+            })
+            .then(() => {
+                return cache.get('a key').should.eventually.contain({hit: false});
+            });
+    });
+
+    it('should handle existing data', () => {
+        const tempDir = newTempDir();
+        fs.writeFileSync(path.join(tempDir, 'abcdef'), 'this is abcdef');
+        fs.mkdirSync(path.join(tempDir, 'path'));
+        fs.writeFileSync(path.join(tempDir, 'path', 'test'), 'this is path/test');
+        const cache = new OnDiskCache(tempDir, 1);
+        return Promise.all([
+            cache.get('abcdef').should.eventually.eql({hit: true, data: new Buffer('this is abcdef')}),
+            cache.get('path/test').should.eventually.eql({hit: true, data: new Buffer('this is path/test')})]);
+    });
+
+    // MRG ideally handle the case of pre-populated stuff overflowing the size
+    // and test sorting by mtime, but that might be too tricky.
+});
+
+const S3FS = {};
+AWS.mock('S3', 'getObject', (params, callback) => {
+    params.Bucket.should.equal("test.bucket");
+    const result = S3FS[params.Key];
+    if (!result) {
+        const error = new Error("Not found");
+        error.code = "NoSuchKey";
+        callback(error);
+    } else {
+        callback(null, {Body:result});
+    }
+});
+AWS.mock('S3', 'putObject', (params, callback) => {
+    params.Bucket.should.equal("test.bucket");
+    S3FS[params.Key] = params.Body;
+    callback(null, {});
+});
+describe('S3 tests', () => {
+    basicTests(() => new S3Cache('test.bucket', 'cache', 'uk-north-1'));
+    // BE VERY CAREFUL - the below can be used with sufficient permissions to test on prod (With mocks off)...
+    // basicTests(() => new S3Cache('storage.godbolt.org', 'cache', 'us-east-1'));
+});