diff --git a/etc/config/aws.amazon.properties b/etc/config/aws.amazon.properties index 923b7a6ed..86a11072e 100644 --- a/etc/config/aws.amazon.properties +++ b/etc/config/aws.amazon.properties @@ -4,3 +4,4 @@ storagePrefix=state storageDynamoTable=links storageBucketArgStats=storage.godbolt.org storagePrefixArgStats=compargs +googleLinksDynamoTable=goo-gl-links diff --git a/lib/app/main.ts b/lib/app/main.ts index eb8b98244..ea9710668 100644 --- a/lib/app/main.ts +++ b/lib/app/main.ts @@ -111,6 +111,7 @@ export async function initialiseApplication(options: ApplicationOptions): Promis const serverDependencies = { ceProps: ceProps, + awsProps: awsProps, sponsorConfig: loadSponsorsFromString( await fs.readFile(path.join(appArgs.rootDir, 'config', 'sponsors.yaml'), 'utf8'), ), diff --git a/lib/app/server-config.ts b/lib/app/server-config.ts index dc4e1c219..2ad7434df 100644 --- a/lib/app/server-config.ts +++ b/lib/app/server-config.ts @@ -143,6 +143,7 @@ export function setupLoggingMiddleware(isDevMode: boolean, router: Router): void * @param renderConfig - Function to render configuration for templates * @param embeddedHandler - Handler for embedded mode * @param ceProps - Compiler Explorer properties + * @param awsProps - AWS properties * @param faviconFilename - Favicon filename * @param options - Server options * @param clientOptionsHandler - Client options handler @@ -152,11 +153,12 @@ export function setupBasicRoutes( renderConfig: RenderConfigFunction, embeddedHandler: express.Handler, ceProps: PropertyGetter, + awsProps: PropertyGetter, faviconFilename: string, options: ServerOptions, clientOptionsHandler: ClientOptionsSource, ): void { - const legacyGoogleUrlHandler = new LegacyGoogleUrlHandler(ceProps); + const legacyGoogleUrlHandler = new LegacyGoogleUrlHandler(ceProps, awsProps); router .use(compression()) diff --git a/lib/app/server.interfaces.ts b/lib/app/server.interfaces.ts index a32bdb3ed..ce81a0dff 100644 --- a/lib/app/server.interfaces.ts +++ b/lib/app/server.interfaces.ts @@ -84,6 +84,7 @@ export interface WebServerResult { export interface ServerDependencies { ceProps: PropertyGetter; + awsProps: PropertyGetter; sponsorConfig: Sponsors; clientOptionsHandler: ClientOptionsSource; storageSolution: string; diff --git a/lib/app/server.ts b/lib/app/server.ts index da8cf3f91..ce7db197e 100644 --- a/lib/app/server.ts +++ b/lib/app/server.ts @@ -80,6 +80,7 @@ export async function setupWebServer( renderConfig, embeddedHandler, dependencies.ceProps, + dependencies.awsProps, getFaviconFilename(appArgs.devMode, appArgs.env), options, dependencies.clientOptionsHandler, diff --git a/lib/app/url-handlers.ts b/lib/app/url-handlers.ts index 6b4d81643..1a00c89f7 100644 --- a/lib/app/url-handlers.ts +++ b/lib/app/url-handlers.ts @@ -47,9 +47,13 @@ export class LegacyGoogleUrlHandler { /** * Create a new handler for legacy Google URL shortcuts * @param ceProps - Compiler Explorer properties + * @param awsProps - AWS properties */ - constructor(private readonly ceProps: PropertyGetter) { - this.googleShortUrlResolver = new ShortLinkResolver(); + constructor( + private readonly ceProps: PropertyGetter, + awsProps: PropertyGetter, + ) { + this.googleShortUrlResolver = new ShortLinkResolver(awsProps); } /** diff --git a/lib/shortener/google.ts b/lib/shortener/google.ts index 643b4d1ef..b03482f49 100644 --- a/lib/shortener/google.ts +++ b/lib/shortener/google.ts @@ -22,32 +22,110 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. +import {DynamoDB} from '@aws-sdk/client-dynamodb'; +import {Counter} from 'prom-client'; + +import {awsCredentials} from '../aws.js'; +import {logger} from '../logger.js'; +import {PropertyGetter} from '../properties.interfaces.js'; + +const GoogleLinkDynamoDbHitCounter = new Counter({ + name: 'ce_google_link_dynamodb_hits_total', + help: 'Total number of successful Google short link lookups from DynamoDB', +}); + +const GoogleLinkDynamoDbMissCounter = new Counter({ + name: 'ce_google_link_dynamodb_misses_total', + help: 'Total number of Google short link lookups not found in DynamoDB', +}); + // This will stop working in August 2025. // https://developers.googleblog.com/en/google-url-shortener-links-will-no-longer-be-available/ export class ShortLinkResolver { - resolve(url: string) { - return new Promise<{longUrl: string}>((resolve, reject) => { - const settings: RequestInit = { - method: 'HEAD', - redirect: 'manual', - }; + private readonly dynamoDb?: DynamoDB; + private readonly tableName?: string; - fetch(url + '?si=1', settings) - .then((res: Response) => { - if (res.status !== 302) { - reject(`Got response ${res.status}`); - return; - } - const targetLocation = res.headers.get('Location'); - if (!targetLocation) { - reject(`Missing location url in ${targetLocation}`); - return; - } - resolve({ - longUrl: targetLocation, - }); - }) - .catch(err => reject(err.message)); + constructor(awsProps?: PropertyGetter) { + if (awsProps) { + const tableName = awsProps('googleLinksDynamoTable', ''); + if (tableName) { + const region = awsProps('region') as string; + this.tableName = tableName; + this.dynamoDb = new DynamoDB({region: region, credentials: awsCredentials()}); + logger.info(`Using DynamoDB table ${tableName} in region ${region} for Google link resolution`); + } + } + } + + async resolve(url: string): Promise<{longUrl: string}> { + const fragment = this.extractFragment(url); + + if (this.hasDynamoDbConfigured() && fragment) { + const dynamoResult = await this.tryDynamoDbLookup(fragment); + if (dynamoResult) { + return dynamoResult; + } + } + + // In August 2025, the Google URL shortener will stop working. At that point, use this code: + // throw new Error('404: Not Found'); + + return this.fallbackToGoogleShortener(url); + } + + // Exported for testing + extractFragment(url: string): string | undefined { + return url.split('/').pop()?.split('?')[0]; + } + + // Exported for testing + hasDynamoDbConfigured(): boolean { + return !!(this.dynamoDb && this.tableName); + } + + private async tryDynamoDbLookup(fragment: string): Promise<{longUrl: string} | null> { + try { + const result = await this.dynamoDb!.getItem({ + TableName: this.tableName!, + Key: { + fragment: {S: fragment}, + }, + }); + + const expandedUrl = result.Item?.expanded_url?.S; + if (expandedUrl) { + GoogleLinkDynamoDbHitCounter.inc(); + return {longUrl: expandedUrl}; + } + + GoogleLinkDynamoDbMissCounter.inc(); + logger.warn(`Google short link '${fragment}' not found in DynamoDB, falling back to Google URL shortener`); + return null; + } catch (err) { + logger.error('DynamoDB lookup failed:', err); + return null; + } + } + + private async fallbackToGoogleShortener(url: string): Promise<{longUrl: string}> { + if (this.hasDynamoDbConfigured()) { + logger.warn(`Falling back to Google URL shortener for '${url}' - this indicates missing data in DynamoDB`); + } + + const res = await fetch(url + '?si=1', { + method: 'HEAD', + redirect: 'manual', }); + + if (res.status !== 302) { + throw new Error(`Got response ${res.status}`); + } + + const targetLocation = res.headers.get('Location'); + if (!targetLocation) { + throw new Error('Missing location url'); + } + + return {longUrl: targetLocation}; } } diff --git a/test/app/rendering-tests.ts b/test/app/rendering-tests.ts index 793225194..e8bfab98e 100644 --- a/test/app/rendering-tests.ts +++ b/test/app/rendering-tests.ts @@ -103,6 +103,7 @@ describe('Rendering Module', () => { getAllTopIcons: vi.fn().mockReturnValue([]), }, ceProps: vi.fn(), + awsProps: vi.fn(), }; }); diff --git a/test/app/server-tests.ts b/test/app/server-tests.ts index e4a322766..27cd5d6fa 100644 --- a/test/app/server-tests.ts +++ b/test/app/server-tests.ts @@ -114,6 +114,7 @@ describe('Server Module', () => { if (key === 'allowedShortUrlHostRe') return '.*'; return ''; }), + awsProps: vi.fn().mockReturnValue(''), healthcheckController: { createRouter: vi.fn().mockReturnValue(express.Router()), }, diff --git a/test/app/url-handlers-tests.ts b/test/app/url-handlers-tests.ts index 7b0ba0216..72c14243a 100644 --- a/test/app/url-handlers-tests.ts +++ b/test/app/url-handlers-tests.ts @@ -64,6 +64,7 @@ describe('Url Handlers', () => { describe('LegacyGoogleUrlHandler', () => { let handler: LegacyGoogleUrlHandler; let mockCeProps: any; + let mockAwsProps: any; let mockRequest: Partial; let mockResponse: Partial; let mockNext: any; @@ -73,7 +74,8 @@ describe('Url Handlers', () => { if (key === 'allowedShortUrlHostRe') return '.*'; return ''; }); - handler = new LegacyGoogleUrlHandler(mockCeProps); + mockAwsProps = vi.fn().mockReturnValue(''); + handler = new LegacyGoogleUrlHandler(mockCeProps, mockAwsProps); // Mock ShortLinkResolver (private property) Object.defineProperty(handler, 'googleShortUrlResolver', { diff --git a/test/google-tests.ts b/test/google-tests.ts index f4b3eb39c..18826607d 100644 --- a/test/google-tests.ts +++ b/test/google-tests.ts @@ -57,7 +57,7 @@ describe('Google short URL resolver tests', () => { it('Handles missing location header', async () => { fetch.mockResponse('', {status: 302}); - await expect(resolver.resolve(googlEndpoint)).rejects.toThrow('Missing location url in null'); + await expect(resolver.resolve(googlEndpoint)).rejects.toThrow('Missing location url'); }); it('Handles failed requests', async () => { @@ -76,3 +76,63 @@ describe('Google short URL resolver tests', () => { }); }); }); + +describe('ShortLinkResolver utility methods', () => { + const resolver = new google.ShortLinkResolver(); + + describe('extractFragment', () => { + it('should extract fragment from goo.gl URL', () => { + expect(resolver.extractFragment('https://goo.gl/abcd1234')).toBe('abcd1234'); + }); + + it('should extract fragment from URL with query parameters', () => { + expect(resolver.extractFragment('https://goo.gl/xyz789?param=value')).toBe('xyz789'); + }); + + it('should handle URL with multiple query parameters', () => { + expect(resolver.extractFragment('https://goo.gl/test123?foo=bar&baz=qux')).toBe('test123'); + }); + + it('should handle URL with fragment and query', () => { + expect(resolver.extractFragment('https://goo.gl/hello?world=123')).toBe('hello'); + }); + + it('should handle simple fragment', () => { + expect(resolver.extractFragment('https://goo.gl/x')).toBe('x'); + }); + + it('should handle URL ending with slash', () => { + expect(resolver.extractFragment('https://goo.gl/fragment/')).toBe(''); + }); + + it('should return empty string for empty URL', () => { + expect(resolver.extractFragment('')).toBe(''); + }); + + it('should return empty string for URL with no path segments', () => { + expect(resolver.extractFragment('https://')).toBe(''); + }); + + it('should handle URL without fragment', () => { + expect(resolver.extractFragment('https://goo.gl/')).toBe(''); + }); + }); + + describe('hasDynamoDbConfigured', () => { + it('should return false for resolver without DynamoDB config', () => { + const basicResolver = new google.ShortLinkResolver(); + expect(basicResolver.hasDynamoDbConfigured()).toBe(false); + }); + + it('should return false for resolver with undefined props', () => { + const basicResolver = new google.ShortLinkResolver(undefined); + expect(basicResolver.hasDynamoDbConfigured()).toBe(false); + }); + + it('should return false for resolver with empty props', () => { + const mockProps = vi.fn().mockReturnValue(undefined); + const basicResolver = new google.ShortLinkResolver(mockProps); + expect(basicResolver.hasDynamoDbConfigured()).toBe(false); + }); + }); +});