From 3b682a12f46cd77e29ab4342a6ae2812fbb2a129 Mon Sep 17 00:00:00 2001 From: fenwick67 Date: Thu, 1 Aug 2019 21:52:10 -0400 Subject: [PATCH] make boosts work. It's kinda slow. --- lib/convertv2.js | 119 ++++++++++++++++++++++++++++++++++---------- npm-shrinkwrap.json | 22 ++++++++ package.json | 1 + 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/lib/convertv2.js b/lib/convertv2.js index db0c791..88a17a9 100644 --- a/lib/convertv2.js +++ b/lib/convertv2.js @@ -4,20 +4,38 @@ var template = ejs.compile(fs.readFileSync('./lib/template.ejs', 'utf8')); var timeAgo = require('timeago.js'); // TODO try https://www.npmjs.com/package/request-promise-cache for the requests -var request = require('request-promise-native') +var request = require('request-promise-cache') + +const hour = 3600000; // get JSON for an AP URL // TODO make it reject on HTTP 4xx or 5xx -async function apGet(url) { - return request.get( { - uri:url, - headers: { - "accept": "application/activity+json" - }, - transform: function (body) { - return JSON.parse(body); - } +async function apGet(url,ttl) { + return new Promise(function(resolve,reject){ + + request( { + uri:url, + cacheKey:url, + cacheTTL:ttl || 24 * hour, + headers: { + "accept": "application/activity+json" + } + }) + .then(body=>JSON.parse(body)) + .then(resolve) + .catch(reject) + }) + +} + +// never rejects, instead it returns an object with a status, error and/or result. +async function apGetNoReject(...args){ + return new Promise(function(resolve,reject){ + apGet(...args) + .then(res=>resolve({status:true,result:res})) + .catch(e=>resolve({status:false,error:e})) + }); } module.exports = async function (opts) { @@ -33,25 +51,25 @@ module.exports = async function (opts) { var user, feed; - // get user and feed in parallel if I have both URLs + // get user and feed in parallel if I have both URLs. + // can cache feed aggressively since it is a specific start and end. if (userUrl && feedUrl){ [user, feed] = await Promise.all([ apGet(userUrl), apGet(feedUrl) ]); }else{ // get user, then outbox, then feed - user = await apGet(userUrl); + user = await apGet(userUrl,24 * hour); isIndex = true; - var outbox = await apGet(user.outbox); + var outbox = await apGet(user.outbox,24 * hour); feedUrl = outbox.first; - feed = await apGet(feedUrl); + feed = await apGet(feedUrl,hour/6);// 10 mins. Because the base feed URL can get new toots quickly. } - - + var templateData = { opts: opts,// from the request meta: metaForUser(user), - items: itemsForFeed(user,feed), + items: await itemsForFeed(opts,user,feed), nextPageLink: getNextPage(opts,user,feed), isIndex: isIndex }; @@ -70,11 +88,59 @@ function metaForUser(user) { } // TODO make function -function itemsForFeed(user,feed) { +async function itemsForFeed(opts,user,feed) { - return feed.orderedItems.filter((item)=>{ + var items = feed.orderedItems; + + if (opts.boosts){ + // yes, I have to fetch all the fucking boosts for this whole feed apparently >:/ + var boostData = []; + var boostUrls = feed.orderedItems.filter(i=>i.type=="Announce").map(i=>i.object); + // console.log(boostUrls); + boostData = await Promise.all(boostUrls.map(apGetNoReject)); + boostData = boostData.map(d=>d.result||{}); + + // now get user data for each of those + let userData = await Promise.all(boostData.map(d=>d.attributedTo||'').map(apGetNoReject)); + + // put a ._userdata key on the item object if this is a boost etc + for (var i = 0; i < boostData.length; i ++){ + if (userData[i].status){ + boostData[i]._userdata = userData[i].result; + } + } + + // some URLs may have failed but IDGAF + + // console.log(boostData[0]); + + boostData.forEach((boostToot)=>{ + + // inject in-place into items + + var index = -1; + for (var i = 0; i < items.length; i ++){ + if (items[i].object == boostToot.id){ + index = i; + break; + } + } + + if (index == -1){ + console.warn("warning: couldn't match boost to item: ",boostToot.id) + return; + } + + boostToot.object = boostToot;// this lets the later stage parser access object without errors :) + items[i] = boostToot; + }) + } + + + return items.filter((item)=>{ // this is temporary, don't handle boosts (TODO) - return item.type == "Create" && item.object && item.object.type=="Note"; + // return item.type == "Create" && item.object && item.object.type=="Note"; + return typeof item.object == 'object';// handle weird cases }).map((item)=>{ var enclosures = (item.object.attachment||[]).filter((a)=>{ @@ -87,10 +153,11 @@ function itemsForFeed(user,feed) { } }); + var op = item._userdata?item._userdata:user; return { - isBoost:false, - title:'New Status by '+user.preferredUsername, + isBoost:!!item._userdata, + title:item._userdata?user.preferredUsername+' shared a status by '+op.preferredUsername:'', isReply:!!(item.object && item.object.inReplyTo), hasCw:item.object.sensitive||false, cw:item.object.summary, @@ -99,10 +166,10 @@ function itemsForFeed(user,feed) { enclosures:enclosures, stringDate:item.published?getTimeDisplay(Date.parse(item.published)):'', author:{ - uri:user.url,// link to author page - avatar:user.icon&&user.icon.url?user.icon.url:'', - displayName:user.name, - fullName:user.preferredUsername+'@'+(new URL(user.url).hostname), + uri:op.url,// link to author page + avatar:op.icon&&op.icon.url?op.icon.url:'', + displayName:op.name, + fullName:op.preferredUsername+'@'+(new URL(op.url).hostname), } } }) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 609f808..014b4a7 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -451,6 +451,11 @@ } } }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "extsprintf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz", @@ -1008,6 +1013,14 @@ "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==", "dev": true }, + "nano-cache": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.2.tgz", + "integrity": "sha512-tqs6EYyIq8oCQMc1L+QSUdZYOk/STwWCmIqyGO6cXI4QhejSb586WPBgO3nplzD5L3+aJCAITMVtMTFFLAdb7w==", + "requires": { + "extend": "~3.0.0" + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -1402,6 +1415,15 @@ } } }, + "request-promise-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-2.0.1.tgz", + "integrity": "sha512-y+me4+M3wpQ1Sj4WPr3Ywg459UjkP+uyh0JlM0FQxjZtNNqixQxbqhmo5M3t+e2mjglMpfm6UPDhSo40kLWsmA==", + "requires": { + "nano-cache": "^1.1.2", + "request": "^2.88.0" + } + }, "request-promise-core": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz", diff --git a/package.json b/package.json index 7ac2ca9..b76b709 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "express": "^4.16.4", "feedparser": "^2.2.9", "request": "^2.88.0", + "request-promise-cache": "^2.0.1", "request-promise-native": "^1.0.7", "serve-static": "^1.13.2", "timeago.js": "^3.0.2"