Consume posts['reddit'] on the unified queue pipeline

.gitignore

@@ -1 +1,2 @@
 node_modules
+.env

index.js

@@ -1,46 +1,56 @@
+require( 'dotenv' ).config();
+
 const Queue = require( 'bull' );
 
-const reddit = require( './indexers/Reddit' );
+const indexers = require( './indexers' );
 
 if ( !process.env.REDIS_URL ) {
     throw new Error( 'Got no queue, exiting' );
 }
 
-const redditQueue = new Queue(
-    'reddit-posts',
+const postsQueue = new Queue(
+    'posts',
     process.env.REDIS_URL,
     {
         limiter: {
             max: 1,
-            duration: 2000, // Might be 2 requests / post (content & parent)
+            duration: 10000,
         },
     }
 );
 
-redditQueue.on( 'error', ( queueError ) => {
+postsQueue.on( 'error', ( queueError ) => {
     console.error( queueError );
 } );
 
-redditQueue.on( 'failed', ( job, jobError ) => {    
+postsQueue.on( 'failed', ( job, jobError ) => {
     // If the API returns duplicate, don't keep it around
     if(jobError.message.includes('returned 409')){
         console.log(`Removed job ${job.id} as the content is a duplicate`);
         job.remove();
 
         return true;
     }
-    
+
     console.error( jobError );
 } );
 
-redditQueue.process( ( job ) => {
-    console.log( `Running job ${ job.id } for ${ job.data.game }` );
+postsQueue.process( 'reddit', ( job ) => {
+    console.log( `Running ${ job.name } job ${ job.id } for ${ job.data.game }` );
+
+    if ( !indexers[ job.name ] ) {
+        console.error( `No indexer specified for ${ job.name }` );
+
+        return Promise.reject();
+    }
 
     if ( !job.data.accountId ) {
         return job.discard();
     }
 
-    return reddit.parsePost( job.data.accountId, job.data.post )
+    const postIndexer = new indexers[ job.name ]( job.data.post.indexerConfig );
+
+    return postIndexer.parsePost( job.data.accountId, job.data.post )
         .then( ( post ) => {
             if ( !post ) {
                 console.log( `Discarding job ${ job.id } because we didn't get a post` );

indexers/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+    reddit: require( './reddit' ),
+};

indexers/Reddit.js → indexers/reddit.js

@@ -194,4 +194,4 @@ class Reddit {
     }
 }
 
-module.exports = new Reddit();
+module.exports = Reddit;

modules/api.js

@@ -3,10 +3,8 @@ const querystring = require( 'querystring' );
 const fs = require( 'fs' );
 const path = require( 'path' );
 
-const API_HOST = 'api.developertracker.com';
-const API_PORT = 443;
-// const API_HOST = 'lvh.me';
-// const API_PORT = 3000;
+const API_HOST = process.env.API_HOST || 'api.developertracker.com';
+const API_PORT = process.env.API_PORT || 443;
 
 const SUCESS_STATUS_CODE = 200;