add retries
continuous-integration/drone/push Build is passing Details

Signed-off-by: Sam Therapy <sam@samtherapy.net>
This commit is contained in:
Sam Therapy 2022-10-28 16:21:49 +02:00
parent 67fb7ff0db
commit 4d554e7095
Signed by: sam
GPG Key ID: 4D8B07C18F31ACBD
14 changed files with 193 additions and 149 deletions

View File

@ -13,8 +13,8 @@
"rules": {
"linebreak-style": ["error", "unix"],
"quotes": ["error", "double"],
"semi": ["error", "always"],
"prettier/prettier": ["error", { "singleQuote": false }]
"semi": ["error", "never"],
"prettier/prettier": ["error", { "singleQuote": false, "semi": false }]
},
"extends": [
"eslint:recommended",

View File

@ -9,6 +9,9 @@
"message": "", // example: "Hello, world!"
"visibility": "unlisted", // example: "public", "unlisted", "private", "direct"
// Misc settings
"retries": 5, // Number of times to retry an upload/post if it fails. Default is 5 if this does not exist.
"remote": false, // **Set this to `true` if you want to serve a file from a booru!**
/* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `false` */
"booru": "safebooru.org", // example: "safebooru.org"

View File

@ -5,7 +5,8 @@
"command-line-usage": "6.1.3",
"got-cjs": "12.3.1",
"json5": "2.2.1",
"megalodon": "4.1.0"
"megalodon": "4.1.0",
"p-retry": "4.6.2"
},
"name": "@froth/fediverse-imagebot",
"version": "2.1.1",
@ -18,8 +19,8 @@
"scripts": {
"build": "tsc -b",
"clean": "tsc -b --clean",
"lint": "eslint --ext .ts ./src --fix && prettier --write ./src",
"lint:ci": "eslint --ext .ts,.js ./src && prettier ./src --check",
"lint": "eslint --ext .ts ./src --fix && prettier --no-semi --write ./src",
"lint:ci": "eslint --ext .ts,.js ./src && prettier --no-semi --check ./src",
"bot": "node ./dist/bot.js",
"package": "pkg . -C Gzip",
"test": "echo \"No tests yet!\" && exit 0",

View File

@ -15,6 +15,7 @@ specifiers:
got-cjs: 12.3.1
json5: 2.2.1
megalodon: 4.1.0
p-retry: 4.6.2
pkg: 5.8.0
prettier: 2.7.1
typescript: 4.8.4
@ -26,6 +27,7 @@ dependencies:
got-cjs: 12.3.1
json5: 2.2.1
megalodon: 4.1.0
p-retry: 4.6.2
devDependencies:
'@types/command-line-args': 5.2.0
@ -219,6 +221,10 @@ packages:
'@types/node': 18.11.7
dev: false
/@types/retry/0.12.0:
resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
dev: false
/@types/semver/7.3.12:
resolution: {integrity: sha512-WwA1MW0++RfXmCr12xeYOOC5baSC9mSb0ZqCquFzKhcoF4TvHu5MKOuXsncgZcpVFhB1pXd5hZmM0ryAoCp12A==}
dev: true
@ -1540,6 +1546,14 @@ packages:
p-limit: 3.1.0
dev: true
/p-retry/4.6.2:
resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==}
engines: {node: '>=8'}
dependencies:
'@types/retry': 0.12.0
retry: 0.13.1
dev: false
/parent-module/1.0.1:
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
engines: {node: '>=6'}
@ -1758,6 +1772,11 @@ packages:
lowercase-keys: 2.0.0
dev: false
/retry/0.13.1:
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
dev: false
/reusify/1.0.4:
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
engines: {iojs: '>=1.0.0', node: '>=0.10.0'}

View File

@ -1,19 +1,19 @@
#!/usr/bin/env node
import getConfig from "./helpers/getConfig.js";
import { config } from "./helpers/types.js";
import getLocalImage from "./getLocalImage.js";
import getRemoteImage from "./getRemoteImage.js";
import getConfig from "./helpers/getConfig.js"
import { config } from "./helpers/types.js"
import getLocalImage from "./getLocalImage.js"
import getRemoteImage from "./getRemoteImage.js"
/**
* Main function
*/
async function main() {
const conf: config = await getConfig();
const conf: config = await getConfig()
if (conf.remote) await getRemoteImage(conf);
else await getLocalImage(conf);
if (conf.remote) await getRemoteImage(conf)
else await getLocalImage(conf)
}
// Run the main function, obviously.
main();
main()

View File

@ -1,10 +1,11 @@
import { createReadStream, ReadStream } from "fs";
import { readdir } from "fs/promises";
import { exit } from "process";
import args from "./helpers/args.js";
import crashHandler from "./helpers/crashHandler.js";
import { config } from "./helpers/types.js";
import postImage from "./postImage.js";
import { createReadStream, ReadStream } from "node:fs"
import { readdir } from "node:fs/promises"
import { exit } from "node:process"
import args from "./helpers/args.js"
import crashHandler from "./helpers/crashHandler.js"
import { config } from "./helpers/types.js"
import postImage from "./postImage.js"
/**
* Get a local image from the filesystem
@ -15,47 +16,47 @@ export default async function getLocalImage(conf: config) {
// Get SFW directory
const sfw_files: string[] = await readdir(`${conf.directory}/sfw`).catch(
(e) => {
crashHandler("Error reading SFW image directory.", e);
return [];
crashHandler("Error reading SFW image directory.", e)
return []
}
);
)
// Get NSFW directory
const nsfw_files: string[] = await readdir(`${conf.directory}/nsfw`).catch(
(e) => {
crashHandler("Error reading NSFW image directory.", e);
return [];
crashHandler("Error reading NSFW image directory.", e)
return []
}
);
)
const random = Math.floor(
Math.random() * (sfw_files.length + nsfw_files.length)
);
)
// Filler that is used to get a random file from the directories
let image: ReadStream;
let sensitivity: boolean;
let file: string;
let image: ReadStream
let sensitivity: boolean
let file: string
if (random >= sfw_files.length) {
// Image is NSFW, mark it sensitive
file = `${conf.directory}/nsfw/${nsfw_files[random - sfw_files.length]}`;
file = `${conf.directory}/nsfw/${nsfw_files[random - sfw_files.length]}`
image = createReadStream(file).on("error", (err: Error) => {
crashHandler(`Error reading file "${file}"`, err);
});
sensitivity = true;
crashHandler(`Error reading file "${file}"`, err)
})
sensitivity = true
} else {
// Image is SFW, mark it not sensitive
file = `${conf.directory}/sfw/${sfw_files[random]}`;
file = `${conf.directory}/sfw/${sfw_files[random]}`
image = createReadStream(file).on("error", (err: Error) => {
crashHandler(`Error reading file "${file}"`, err);
});
sensitivity = false;
crashHandler(`Error reading file "${file}"`, err)
})
sensitivity = false
}
if (args.verbose) {
console.log(`File being sent: ${file}`);
console.log(`Sensitivity: ${sensitivity}`);
console.log(`File being sent: ${file}`)
console.log(`Sensitivity: ${sensitivity}`)
}
await postImage(image, sensitivity, conf);
exit(0);
await postImage(image, sensitivity, conf)
exit(0)
}

View File

@ -1,15 +1,15 @@
import { search } from "booru";
import Post from "booru/dist/structures/Post"; // Ce n'est pas bien
import { createReadStream, createWriteStream } from "fs";
import { unlink } from "fs/promises";
import got from "got-cjs";
import stream from "node:stream";
import { promisify } from "node:util";
import { exit } from "process";
import args from "./helpers/args.js";
import crashHandler from "./helpers/crashHandler.js";
import { config } from "./helpers/types.js";
import postImage from "./postImage.js";
import { createReadStream, createWriteStream } from "node:fs"
import { unlink } from "node:fs/promises"
import { exit } from "node:process"
import stream from "node:stream"
import { promisify } from "node:util"
import { search, Post } from "booru"
import got from "got-cjs"
import args from "./helpers/args.js"
import crashHandler from "./helpers/crashHandler.js"
import { config } from "./helpers/types.js"
import postImage from "./postImage.js"
/**
* Get a remote image from a booru
@ -23,43 +23,43 @@ export default async function getRemoteImage(conf: config) {
limit: 1,
random: true,
}).catch((e) => {
crashHandler("Error searching for posts.", e);
return [] as Post[];
});
crashHandler("Error searching for posts.", e)
return [] as Post[]
})
if (searchResults.length === 0) {
crashHandler("Error searching for posts.", Error("No posts found."));
return;
crashHandler("Error searching for posts.", Error("No posts found."))
return
}
const post = searchResults[0];
if (args.verbose) console.log(`Found post: ${post.id} at ${post.file_url}`);
const post = searchResults[0]
if (args.verbose) console.log(`Found post: ${post.id} at ${post.file_url}`)
// Set the post as sensitive if the rating is not safe
const sensitivity: boolean = post.rating !== "s";
const sensitivity: boolean = post.rating !== "s"
// Make an HTTP request for the image
const filename: string = post.fileUrl?.split("/").pop() as string; // Type checks for type checks
const pipeline = promisify(stream.pipeline);
const filename: string = post.fileUrl?.split("/").pop() as string // Type checks for type checks
const pipeline = promisify(stream.pipeline)
// Make the HTTP request as a stream so it can be piped to the file system
await pipeline(
got.stream(post.file_url as string),
createWriteStream(filename)
).catch((err: Error) => {
crashHandler("Error saving downloading image.", err);
});
if (args.verbose) console.log(`Saved image to ${filename}`);
crashHandler("Error saving downloading image.", err)
})
if (args.verbose) console.log(`Saved image to ${filename}`)
const str = createReadStream(filename).on("error", (err: Error) => {
crashHandler("Error reading downloaded image.", err);
});
crashHandler("Error reading downloaded image.", err)
})
if (args.verbose) {
console.log(`File being sent: ${filename}\nSensitivity: ${sensitivity}`);
console.log(`File being sent: ${filename}\nSensitivity: ${sensitivity}`)
}
// Make a status with the image
await postImage(str, sensitivity, conf);
await postImage(str, sensitivity, conf)
// Delete the image that it downloaded
await unlink(filename).catch((err: Error) => {
crashHandler("Error deleting downloaded image.", err);
});
if (args.verbose) console.log(`Successfully deleted image ${filename}`);
exit(0);
crashHandler("Error deleting downloaded image.", err)
})
if (args.verbose) console.log(`Successfully deleted image ${filename}`)
exit(0)
}

View File

@ -1,7 +1,8 @@
import commandLineArgs from "command-line-args";
import commandLineUsage from "command-line-usage";
import { exit } from "process";
import writeConfig from "./writeconfig.js";
import { exit } from "node:process"
import commandLineArgs from "command-line-args"
import commandLineUsage from "command-line-usage"
import writeConfig from "./writeconfig.js"
const optionDefinitions = [
{
@ -24,7 +25,7 @@ const optionDefinitions = [
description:
"Path to the JSON configuration file. (default: ./config.jsonc)",
defaultValue: "./config.jsonc",
typeLabel: "<file.json[c]>",
typeLabel: "<file.json[c,5]>",
},
{
name: "writeConfig",
@ -33,9 +34,9 @@ const optionDefinitions = [
description:
"Write a default configuration file to the current directory and exit.",
},
];
]
const args = commandLineArgs(optionDefinitions);
const args = commandLineArgs(optionDefinitions)
if (args.help) {
const usage = commandLineUsage([
@ -52,17 +53,18 @@ if (args.help) {
content:
"Project home: {underline https://git.froth.zone/Sam/fediverse-imagebot}",
},
]);
console.log(usage);
exit(0);
])
console.log(usage)
exit(0)
}
if (args.verbose) console.log("Running in verbose mode.\n");
if (args.verbose) console.log("Running in verbose mode.\n")
if (args.writeConfig) {
writeConfig(args.verbose);
console.log("Wrote default config file to ./config.jsonc");
exit(0);
writeConfig(args.verbose).then(() => {
console.log("Wrote default config file to ./config.jsonc")
exit(0)
})
}
export default args;
export default args

View File

@ -1,5 +1,6 @@
import { exit } from "process";
import args from "./args.js";
import { exit } from "node:process"
import args from "./args.js"
/**
* The function that gets called when the program runs into an error.
@ -8,10 +9,10 @@ import args from "./args.js";
* @returns This function will never return.
*/
export default function crashHandler(msg: string, e: Error, res?: string) {
console.error(`${msg}: ${e.name}`);
console.error(`${msg}: ${e.name}`)
if (args.verbose) {
console.error(`--BEGIN FULL ERROR--\n${e}\n${res}\n--END FULL ERROR--`);
} else console.error("Run with -v to see the full error.");
console.error(`--BEGIN FULL ERROR--\n${e}\n${res}\n--END FULL ERROR--`)
} else console.error("Run with -v to see the full error.")
exit(1);
exit(1)
}

View File

@ -1,8 +1,9 @@
import { readFile } from "fs/promises";
import JSON5 from "json5";
import args from "./args.js";
import crashHandler from "./crashHandler.js";
import { config } from "./types.js";
import { readFile } from "node:fs/promises"
import JSON5 from "json5"
import args from "./args.js"
import crashHandler from "./crashHandler.js"
import { config } from "./types.js"
/**
* Reads the config file and returns the config object
@ -10,9 +11,9 @@ import { config } from "./types.js";
*/
async function readConfig(): Promise<string> {
return readFile(args.config, "utf8").catch((err) => {
crashHandler("Error reading config file.", err);
return "CRASH";
});
crashHandler("Error reading config file.", err)
return "CRASH"
})
}
/**
* Parses the config file and returns it as a JSON object
@ -25,20 +26,23 @@ async function readConfig(): Promise<string> {
* // Prints "https://mastodon.social"
*/
export default async function getConfig(): Promise<config> {
let conf: config;
let conf: config
try {
conf = JSON5.parse(
await readConfig().catch((err) => {
crashHandler("Error reading config file.", err);
return "";
crashHandler("Error reading config file.", err)
return ""
})
);
)
// Backwards compatibility for older versions
conf.retries ??= 5
if (conf.retries < 1) conf.retries = 1
} catch (err: unknown) {
crashHandler("Error parsing config file.", Error(err as string));
return {} as config;
crashHandler("Error parsing config file.", Error(err as string))
return {} as config
}
if (args.verbose) {
console.log(`Read config file: ${args.config}\n${JSON.stringify(conf)}`);
console.log(`Read config file: ${args.config}\n${JSON.stringify(conf)}`)
}
return conf;
return conf
}

View File

@ -2,14 +2,15 @@
* Config type to reduce boilerplate
*/
export type config = {
instance: string;
type: "misskey" | "mastodon" | "pleroma";
accessToken: string;
refreshToken: string | null;
message: string;
visibility: "direct" | "unlisted" | "private" | "public";
remote: boolean;
booru: string;
tags: string[];
directory: string;
};
instance: string
type: "misskey" | "mastodon" | "pleroma"
accessToken: string
refreshToken: string | null
message: string
visibility: "direct" | "unlisted" | "private" | "public"
remote: boolean
booru: string
tags: string[]
directory: string
retries: number
}

View File

@ -1,15 +1,14 @@
import { writeFileSync } from "fs";
import { writeFile } from "node:fs/promises"
/**
* Writes the sample config file to disk
* @returns Nothing
*/
export default function writeConfig(verbose: boolean) {
if (verbose) console.log("Writing sample config to config.jsonc");
writeFileSync(
export default async function writeConfig(verbose: boolean) {
if (verbose) console.log("Writing sample config to config.jsonc")
await writeFile(
"./config.jsonc",
// eslint-disable-next-line prettier/prettier
// prettier-ignore
"{ \n //Instance and token settings \n \"instance\": \"INSTANCE_URL\", // example https://test.com \n \"type\": \"INSTANCE_TYPE\", // examples: \"mastodon\", \"misskey\", \"pleroma\" \n \"accessToken\": \"ACCESS_TOKEN\", // Get a token from https://git.froth.zone/Sam/js-feditoken \n \"refreshToken\": \"REFRESH_TOKEN\", // optional \n \n // Post settings \n \"message\": \"\", // example: \"Hello, world!\" \n \"visibility\": \"unlisted\", // example: \"public\", \"unlisted\", \"private\", \"direct\" \n \n \"remote\": false, // **Set this to `true` if you want to serve a file from a booru!** \n /* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `false` */ \n \"booru\": \"safebooru.org\", // example: \"safebooru.org\" \n \"tags\": [\"\"], // example: [\"tohsaka_rin\", \"-feet\"] \n \"rating\": \"safe\", // example: \"safe\", \"questionable\", \"explicit\" \n /* END OF SETTINGS THAT WILL BE IGNORED IF YOU SET `remote` TO `false` */ \n \n /* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `true` */ \n \"directory\": \"./images\" // example: \"./images\" \n /* \n Directory structure should be as follows: \n folder/ \n - sfw/ \n - image1.jpg \n - nsfw/ \n - image1.jpg \n */ \n /* END OF SETTINGS THAT WILL BE IGNORED IF YOU SET `remote` TO `true` */ \n} \n"
);
return;
)
}

View File

@ -1,8 +1,10 @@
import { ReadStream } from "fs";
import generator, { Entity, Response } from "megalodon";
import { Readable } from "stream";
import crashHandler from "./helpers/crashHandler.js";
import { config } from "./helpers/types.js";
import { ReadStream } from "node:fs"
import { Readable } from "node:stream"
import generator, { Entity, Response } from "megalodon"
import pRetry from "p-retry"
import crashHandler from "./helpers/crashHandler.js"
import { config } from "./helpers/types.js"
/**
* Uploads an image to a fediverse instance
@ -22,26 +24,37 @@ export default async function postImage(
cfg.instance,
cfg.accessToken,
cfg.refreshToken
);
)
// Upload the image
const res: Response<Entity.Attachment> = await client
.uploadMedia(image)
.catch((err) => {
crashHandler("Error uploading image.", err, err.response.data);
return {} as Response<Entity.Attachment>;
});
const upload = async () => client.uploadMedia(image)
const res = await pRetry(upload, {
retries: cfg.retries,
onFailedAttempt: logRetry,
}).catch((err) => {
crashHandler("Error uploading image.", err, err.response.data)
return {} as Response<Entity.Attachment>
})
// Make a status with the image
await client
.postStatus(cfg.message, {
// Make the post
const post = async () =>
client.postStatus(cfg.message, {
media_ids: [res.data.id],
// Change this to make the post visibility you wish
visibility: cfg.visibility,
sensitive: sensitivity,
})
.catch((err: Error) => {
crashHandler("Error posting status.", err);
});
console.log(`Successfully posted to ${cfg.instance}`);
await pRetry(post, {
retries: cfg.retries,
onFailedAttempt: logRetry,
}).catch((err) => {
crashHandler("Error uploading image", err, err.response.data)
})
console.log(`Successfully posted to ${cfg.instance}`)
}
function logRetry(error: unknown) {
console.error("Retrying, error:")
console.error(error)
}

View File

@ -2,7 +2,7 @@
"compilerOptions": {
/* Basic Options */
"target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */,
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
"module": "CommonJS" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
"lib": [
"es6"
] /* Specify library files to be included in the compilation. */,