make boosts work. It's kinda slow.

lib/convertv2.js

@@ -4,20 +4,38 @@ var template = ejs.compile(fs.readFileSync('./lib/template.ejs', 'utf8'));
 var timeAgo = require('timeago.js');
 
 // TODO try https://www.npmjs.com/package/request-promise-cache for the requests
-var request = require('request-promise-native')
+var request = require('request-promise-cache')
+
+const hour = 3600000;
 
 // get JSON for an AP URL
 // TODO make it reject on HTTP 4xx or 5xx
-async function apGet(url) {
-    return request.get( {
-        uri:url,
-        headers: {
-            "accept": "application/activity+json"
-        },
-        transform: function (body) {
-            return JSON.parse(body);
-        }
+async function apGet(url,ttl) {
+    return new Promise(function(resolve,reject){
+
+        request( {
+            uri:url,
+            cacheKey:url,
+            cacheTTL:ttl || 24 * hour,
+            headers: {
+                "accept": "application/activity+json"
+            }
+        })
+        .then(body=>JSON.parse(body))
+        .then(resolve)
+        .catch(reject)
+
     })
+    
+}
+
+// never rejects, instead it returns an object with a status, error and/or result.
+async function apGetNoReject(...args){
+    return new Promise(function(resolve,reject){
+        apGet(...args)
+            .then(res=>resolve({status:true,result:res}))
+            .catch(e=>resolve({status:false,error:e}))
+    });
 }
 
 module.exports = async function (opts) {
@@ -33,25 +51,25 @@ module.exports = async function (opts) {
 
     var user, feed;
 
-    // get user and feed in parallel if I have both URLs
+    // get user and feed in parallel if I have both URLs.
+    // can cache feed aggressively since it is a specific start and end.
     if (userUrl && feedUrl){
         [user, feed] = await Promise.all([ apGet(userUrl), apGet(feedUrl) ]);
     }else{
         // get user, then outbox, then feed
     
-        user = await apGet(userUrl);
+        user = await apGet(userUrl,24 * hour);
         isIndex = true;
-        var outbox = await apGet(user.outbox);
+        var outbox = await apGet(user.outbox,24 * hour);
         feedUrl = outbox.first;
-        feed = await apGet(feedUrl);
+        feed = await apGet(feedUrl,hour/6);// 10 mins. Because the base feed URL can get new toots quickly.
 
     }
-
-
+    
     var templateData = {
         opts: opts,// from the request
         meta: metaForUser(user),
-        items: itemsForFeed(user,feed),
+        items: await itemsForFeed(opts,user,feed),
         nextPageLink: getNextPage(opts,user,feed),
         isIndex: isIndex
     };
@@ -70,11 +88,59 @@ function metaForUser(user) {
 }
 
 // TODO make function
-function itemsForFeed(user,feed) {
+async function itemsForFeed(opts,user,feed) {
 
-    return feed.orderedItems.filter((item)=>{
+    var items = feed.orderedItems;
+    
+    if (opts.boosts){
+        // yes, I have to fetch all the fucking boosts for this whole feed apparently >:/
+        var boostData = [];
+        var boostUrls = feed.orderedItems.filter(i=>i.type=="Announce").map(i=>i.object);
+        // console.log(boostUrls);
+        boostData = await Promise.all(boostUrls.map(apGetNoReject));
+        boostData = boostData.map(d=>d.result||{});
+
+        // now get user data for each of those
+        let userData = await Promise.all(boostData.map(d=>d.attributedTo||'').map(apGetNoReject));
+
+        // put a ._userdata key on the item object if this is a boost etc
+        for (var i = 0; i < boostData.length; i ++){
+            if (userData[i].status){
+                boostData[i]._userdata = userData[i].result;
+            }
+        }
+
+        // some URLs may have failed but IDGAF
+
+        // console.log(boostData[0]);
+
+        boostData.forEach((boostToot)=>{
+            
+            // inject in-place into items
+
+            var index = -1;
+            for (var i = 0; i < items.length; i ++){
+                if (items[i].object == boostToot.id){
+                    index = i;
+                    break;
+                }
+            }
+
+            if (index == -1){
+                console.warn("warning: couldn't match boost to item: ",boostToot.id)
+                return;
+            }
+
+            boostToot.object = boostToot;// this lets the later stage parser access object without errors :)
+            items[i] = boostToot;
+        })
+    }
+
+
+    return items.filter((item)=>{
         // this is temporary, don't handle boosts (TODO)
-        return item.type == "Create" && item.object && item.object.type=="Note";
+        // return item.type == "Create" && item.object && item.object.type=="Note";
+        return typeof item.object == 'object';// handle weird cases
     }).map((item)=>{
 
         var enclosures = (item.object.attachment||[]).filter((a)=>{
@@ -87,10 +153,11 @@ function itemsForFeed(user,feed) {
             }
         });
 
+        var op = item._userdata?item._userdata:user;
 
         return {
-            isBoost:false,
-            title:'New Status by '+user.preferredUsername,
+            isBoost:!!item._userdata,
+            title:item._userdata?user.preferredUsername+' shared a status by '+op.preferredUsername:'',
             isReply:!!(item.object && item.object.inReplyTo),
             hasCw:item.object.sensitive||false,
             cw:item.object.summary,
@@ -99,10 +166,10 @@ function itemsForFeed(user,feed) {
             enclosures:enclosures,
             stringDate:item.published?getTimeDisplay(Date.parse(item.published)):'',
             author:{
-                uri:user.url,// link to author page
-                avatar:user.icon&&user.icon.url?user.icon.url:'',
-                displayName:user.name,
-                fullName:user.preferredUsername+'@'+(new URL(user.url).hostname),
+                uri:op.url,// link to author page
+                avatar:op.icon&&op.icon.url?op.icon.url:'',
+                displayName:op.name,
+                fullName:op.preferredUsername+'@'+(new URL(op.url).hostname),
             }
         }
     })

npm-shrinkwrap.json

@@ -451,6 +451,11 @@
         }
       }
     },
+    "extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
+    },
     "extsprintf": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz",
@@ -1008,6 +1013,14 @@
       "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==",
       "dev": true
     },
+    "nano-cache": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.2.tgz",
+      "integrity": "sha512-tqs6EYyIq8oCQMc1L+QSUdZYOk/STwWCmIqyGO6cXI4QhejSb586WPBgO3nplzD5L3+aJCAITMVtMTFFLAdb7w==",
+      "requires": {
+        "extend": "~3.0.0"
+      }
+    },
     "negotiator": {
       "version": "0.6.2",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@@ -1402,6 +1415,15 @@
         }
       }
     },
+    "request-promise-cache": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-2.0.1.tgz",
+      "integrity": "sha512-y+me4+M3wpQ1Sj4WPr3Ywg459UjkP+uyh0JlM0FQxjZtNNqixQxbqhmo5M3t+e2mjglMpfm6UPDhSo40kLWsmA==",
+      "requires": {
+        "nano-cache": "^1.1.2",
+        "request": "^2.88.0"
+      }
+    },
     "request-promise-core": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz",

package.json

@@ -5,6 +5,7 @@
     "express": "^4.16.4",
     "feedparser": "^2.2.9",
     "request": "^2.88.0",
+    "request-promise-cache": "^2.0.1",
     "request-promise-native": "^1.0.7",
     "serve-static": "^1.13.2",
     "timeago.js": "^3.0.2"