Enable aggregated Requests

package.json

@@ -39,6 +39,7 @@
     "inert": "^5.1.2",
     "joi": "^14.3.0",
     "jsdom": "^13.1.0",
+    "lodash.chunk": "^4.2.0",
     "repl.history": "^0.1.4",
     "vision": "^5.4.4"
   },

services/util/client.js

@@ -1,13 +1,18 @@
+const assert = require('assert');
 const axios = require('axios');
+const chunk = require('lodash.chunk');
 const Url = require('url');
 
 const errors = require('./errors');
 
 const defaultParams = { action: 'query', format: 'json' };
 const apiPath = 'w/api.php';
+const maxTitles = 50;
 
 function transform(data) {
   const { query } = data;
+  assert.ok(query, errors.emptyResponse);
+  assert.ok(query.pages, errors.emptyResponse);
   return query;
 }
 
@@ -20,12 +25,25 @@ async function queryApi({ client, wikiUrl, params }) {
   const apiUrl = Url.resolve(wikiUrl, apiPath);
   try {
     const { data } = await client.get(apiUrl, { params });
-    return transform(data);
+    return data;
   } catch (error) {
     return handleError(error);
   }
 }
 
+async function batchRequest({ batch, client, wikiUrl, params }) {
+  const titles = batch.join('|');
+  const data = await queryApi({ client, wikiUrl, params: { ...params, titles } });
+  assert.ok(data, errors.emptyResponse);
+  return transform(data);
+}
+
+function mergePages(responses) {
+  return responses
+    .map(response => response.pages)
+    .reduce((pages, page) => Object.assign(pages, page));
+}
+
 class Client {
   constructor() {
     this.client = axios.create({
@@ -39,16 +57,18 @@ class Client {
     });
   }
 
-  getResultsFromApi(titles, prop, wikiUrl, params = {}) {
+  async getResultsFromApi(titles, prop, wikiUrl, params = {}) {
     const { client } = this;
-    const titleString = titles.join('|');
+    const titleBatches = chunk(titles, maxTitles);
     const queryParams = {
       ...defaultParams,
       ...params,
-      titles: titleString,
       prop,
     };
-    return queryApi({ client, wikiUrl, params: queryParams });
+    const responses = await axios.all(
+      titleBatches.map(batch => batchRequest({ batch, client, wikiUrl, params: queryParams }))
+    );
+    return { pages: mergePages(responses) };
   }
 }
 

services/util/client.test.js

@@ -8,7 +8,15 @@ jest.mock('axios');
 describe('Client', () => {
   const axiosClient = { get: jest.fn() };
 
-  beforeEach(() => axios.create.mockReturnValue(axiosClient));
+  beforeEach(() => {
+    axios.create.mockReturnValue(axiosClient);
+    axios.all.mockImplementation(array => Promise.all(array));
+  });
+
+  afterEach(() => {
+    axiosClient.get.mockReset();
+    axios.all.mockReset();
+  });
 
   it('initializes a new axios client with defaults for header and timeout', () => {
     const headers = {
@@ -28,7 +36,7 @@ describe('Client', () => {
     const wikiUrl = 'https://en.wikipedia.org';
     const apiUrl = 'https://en.wikipedia.org/w/api.php';
     const defaultParams = { action: 'query', format: 'json' };
-    const mockedResponse = { data: { query: { foo: 'bar' } } };
+    const mockedResponse = { data: { query: { pages: { foo: 'bar' } } } };
 
     it('returns an error if the API cannot be reached', async () => {
       const titles = ['Def_Leppard'];
@@ -53,6 +61,26 @@ describe('Client', () => {
       await expect(client.getResultsFromApi(titles, 'image', wikiUrl)).rejects.toThrow();
     });
 
+    it('raises an error if the response does not include query object', async () => {
+      const titles = ['Def_Leppard'];
+      axiosClient.get.mockResolvedValue({ data: {} });
+      const client = new Client();
+
+      await expect(client.getResultsFromApi(titles, 'image', wikiUrl)).rejects.toThrow(
+        errors.emptyResponse
+      );
+    });
+
+    it('raises an error if the response does not include pages', async () => {
+      const titles = ['Def_Leppard'];
+      axiosClient.get.mockResolvedValue({ data: { query: { namespace: 'some' } } });
+      const client = new Client();
+
+      await expect(client.getResultsFromApi(titles, 'image', wikiUrl)).rejects.toThrow(
+        errors.emptyResponse
+      );
+    });
+
     it('allows querying for images of a page', async () => {
       const titleString = 'Def_Leppard';
       const titles = [titleString];
@@ -65,7 +93,7 @@ describe('Client', () => {
       const subject = await client.getResultsFromApi(titles, 'image', wikiUrl);
 
       expect(axiosClient.get).toHaveBeenCalledWith(apiUrl, { params });
-      expect(subject).toEqual({ foo: 'bar' });
+      expect(subject).toEqual({ pages: { foo: 'bar' } });
     });
 
     it('allows querying for multiple titles with additional params', async () => {
@@ -83,7 +111,40 @@ describe('Client', () => {
       });
 
       expect(axiosClient.get).toHaveBeenCalledWith(apiUrl, { params });
-      expect(subject).toEqual({ foo: 'bar' });
+      expect(subject).toEqual({ pages: { foo: 'bar' } });
+    });
+
+    it('combines the results of multiple requests if the number of titles is too many', async () => {
+      const titles = [...Array(55).keys()];
+      const prop = 'imageInfo';
+      const batch1 =
+        '0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49';
+      const batch2 = '50|51|52|53|54';
+      const baseParams = { ...defaultParams, prop, iiprop: 'url' };
+      const params1 = { ...baseParams, titles: batch1 };
+      const params2 = { ...baseParams, titles: batch2 };
+
+      axiosClient.get = jest
+        .fn()
+        .mockImplementation((url, { params }) =>
+          Promise.resolve({ data: { query: { pages: { [params.titles]: {} } } } })
+        );
+
+      const client = new Client();
+      const subject = await client.getResultsFromApi(titles, prop, wikiUrl, {
+        titles,
+        iiprop: 'url',
+      });
+
+      expect(axiosClient.get).toHaveBeenCalledWith(apiUrl, { params: params1 });
+      expect(axiosClient.get).toHaveBeenCalledWith(apiUrl, { params: params2 });
+
+      expect(subject).toEqual({
+        pages: {
+          [batch1]: {},
+          [batch2]: {},
+        },
+      });
     });
   });
 });

yarn.lock

@@ -3576,6 +3576,10 @@ lodash-es@^4.17.11:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.11.tgz#145ab4a7ac5c5e52a3531fb4f310255a152b4be0"
   integrity sha512-DHb1ub+rMjjrxqlB3H56/6MXtm1lSksDp2rA2cNWjG8mlDUYFhUj3Di2Zn5IwSU87xLv8tNIQ7sSwE/YOX/D/Q==
+lodash.chunk@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.chunk/-/lodash.chunk-4.2.0.tgz#66e5ce1f76ed27b4303d8c6512e8d1216e8106bc"
+  integrity sha1-ZuXOH3btJ7QwPYxlEujRIW6BBrw=
 
 lodash.debounce@^4.0.8:
   version "4.0.8"