add retries

Signed-off-by: Sam Therapy <sam@samtherapy.net>
2022-10-28 16:21:49 +02:00 · 2022-10-28 16:21:49 +02:00 · 4d554e7095
commit 4d554e7095
parent 67fb7ff0db
14 changed files with 193 additions and 149 deletions
--- a/.eslintrc.json
+++ b/.eslintrc.json
@ -13,8 +13,8 @@
  "rules": {
    "linebreak-style": ["error", "unix"],
    "quotes": ["error", "double"],
-    "semi": ["error", "always"],
-    "prettier/prettier": ["error", { "singleQuote": false }]
+    "semi": ["error", "never"],
+    "prettier/prettier": ["error", { "singleQuote": false, "semi": false }]
  },
  "extends": [
    "eslint:recommended",
--- a/config.sample.jsonc
+++ b/config.sample.jsonc
@ -9,6 +9,9 @@
  "message": "", // example: "Hello, world!"
  "visibility": "unlisted", // example: "public", "unlisted", "private", "direct"

+  // Misc settings
+  "retries": 5, // Number of times to retry an upload/post if it fails. Default is 5 if this does not exist.
+
  "remote": false, // **Set this to `true` if you want to serve a file from a booru!**
  /* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `false` */
  "booru": "safebooru.org", // example: "safebooru.org"
--- a/package.json
+++ b/package.json
@ -5,7 +5,8 @@
    "command-line-usage": "6.1.3",
    "got-cjs": "12.3.1",
    "json5": "2.2.1",
-    "megalodon": "4.1.0"
+    "megalodon": "4.1.0",
+    "p-retry": "4.6.2"
  },
  "name": "@froth/fediverse-imagebot",
  "version": "2.1.1",
@ -18,8 +19,8 @@
  "scripts": {
    "build": "tsc -b",
    "clean": "tsc -b --clean",
-    "lint": "eslint --ext .ts ./src --fix && prettier --write ./src",
-    "lint:ci": "eslint --ext .ts,.js ./src && prettier ./src --check",
+    "lint": "eslint --ext .ts ./src --fix && prettier --no-semi --write ./src",
+    "lint:ci": "eslint --ext .ts,.js ./src && prettier --no-semi --check ./src",
    "bot": "node ./dist/bot.js",
    "package": "pkg . -C Gzip",
    "test": "echo \"No tests yet!\" && exit 0",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -15,6 +15,7 @@ specifiers:
  got-cjs: 12.3.1
  json5: 2.2.1
  megalodon: 4.1.0
+  p-retry: 4.6.2
  pkg: 5.8.0
  prettier: 2.7.1
  typescript: 4.8.4
@ -26,6 +27,7 @@ dependencies:
  got-cjs: 12.3.1
  json5: 2.2.1
  megalodon: 4.1.0
+  p-retry: 4.6.2

 devDependencies:
  '@types/command-line-args': 5.2.0
@ -219,6 +221,10 @@ packages:
      '@types/node': 18.11.7
    dev: false

+  /@types/retry/0.12.0:
+    resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
+    dev: false
+
  /@types/semver/7.3.12:
    resolution: {integrity: sha512-WwA1MW0++RfXmCr12xeYOOC5baSC9mSb0ZqCquFzKhcoF4TvHu5MKOuXsncgZcpVFhB1pXd5hZmM0ryAoCp12A==}
    dev: true
@ -1540,6 +1546,14 @@ packages:
      p-limit: 3.1.0
    dev: true

+  /p-retry/4.6.2:
+    resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==}
+    engines: {node: '>=8'}
+    dependencies:
+      '@types/retry': 0.12.0
+      retry: 0.13.1
+    dev: false
+
  /parent-module/1.0.1:
    resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
    engines: {node: '>=6'}
@ -1758,6 +1772,11 @@ packages:
      lowercase-keys: 2.0.0
    dev: false

+  /retry/0.13.1:
+    resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
+    engines: {node: '>= 4'}
+    dev: false
+
  /reusify/1.0.4:
    resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
    engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
--- a/src/bot.ts
+++ b/src/bot.ts
@ -1,19 +1,19 @@
 #!/usr/bin/env node

-import getConfig from "./helpers/getConfig.js";
-import { config } from "./helpers/types.js";
-import getLocalImage from "./getLocalImage.js";
-import getRemoteImage from "./getRemoteImage.js";
+import getConfig from "./helpers/getConfig.js"
+import { config } from "./helpers/types.js"
+import getLocalImage from "./getLocalImage.js"
+import getRemoteImage from "./getRemoteImage.js"

 /**
 * Main function
 */
 async function main() {
-  const conf: config = await getConfig();
+  const conf: config = await getConfig()

-  if (conf.remote) await getRemoteImage(conf);
-  else await getLocalImage(conf);
+  if (conf.remote) await getRemoteImage(conf)
+  else await getLocalImage(conf)
 }

 // Run the main function, obviously.
-main();
+main()
--- a/src/getLocalImage.ts
+++ b/src/getLocalImage.ts
@ -1,10 +1,11 @@
-import { createReadStream, ReadStream } from "fs";
-import { readdir } from "fs/promises";
-import { exit } from "process";
-import args from "./helpers/args.js";
-import crashHandler from "./helpers/crashHandler.js";
-import { config } from "./helpers/types.js";
-import postImage from "./postImage.js";
+import { createReadStream, ReadStream } from "node:fs"
+import { readdir } from "node:fs/promises"
+import { exit } from "node:process"
+
+import args from "./helpers/args.js"
+import crashHandler from "./helpers/crashHandler.js"
+import { config } from "./helpers/types.js"
+import postImage from "./postImage.js"

 /**
 * Get a local image from the filesystem
@ -15,47 +16,47 @@ export default async function getLocalImage(conf: config) {
  // Get SFW directory
  const sfw_files: string[] = await readdir(`${conf.directory}/sfw`).catch(
    (e) => {
-      crashHandler("Error reading SFW image directory.", e);
-      return [];
+      crashHandler("Error reading SFW image directory.", e)
+      return []
    }
-  );
+  )
  // Get NSFW directory
  const nsfw_files: string[] = await readdir(`${conf.directory}/nsfw`).catch(
    (e) => {
-      crashHandler("Error reading NSFW image directory.", e);
-      return [];
+      crashHandler("Error reading NSFW image directory.", e)
+      return []
    }
-  );
+  )
  const random = Math.floor(
    Math.random() * (sfw_files.length + nsfw_files.length)
-  );
+  )

  // Filler that is used to get a random file from the directories
-  let image: ReadStream;
-  let sensitivity: boolean;
-  let file: string;
+  let image: ReadStream
+  let sensitivity: boolean
+  let file: string

  if (random >= sfw_files.length) {
    // Image is NSFW, mark it sensitive
-    file = `${conf.directory}/nsfw/${nsfw_files[random - sfw_files.length]}`;
+    file = `${conf.directory}/nsfw/${nsfw_files[random - sfw_files.length]}`
    image = createReadStream(file).on("error", (err: Error) => {
-      crashHandler(`Error reading file "${file}"`, err);
-    });
-    sensitivity = true;
+      crashHandler(`Error reading file "${file}"`, err)
+    })
+    sensitivity = true
  } else {
    // Image is SFW, mark it not sensitive
-    file = `${conf.directory}/sfw/${sfw_files[random]}`;
+    file = `${conf.directory}/sfw/${sfw_files[random]}`
    image = createReadStream(file).on("error", (err: Error) => {
-      crashHandler(`Error reading file "${file}"`, err);
-    });
-    sensitivity = false;
+      crashHandler(`Error reading file "${file}"`, err)
+    })
+    sensitivity = false
  }

  if (args.verbose) {
-    console.log(`File being sent: ${file}`);
-    console.log(`Sensitivity: ${sensitivity}`);
+    console.log(`File being sent: ${file}`)
+    console.log(`Sensitivity: ${sensitivity}`)
  }

-  await postImage(image, sensitivity, conf);
-  exit(0);
+  await postImage(image, sensitivity, conf)
+  exit(0)
 }
--- a/src/getRemoteImage.ts
+++ b/src/getRemoteImage.ts
@ -1,15 +1,15 @@
-import { search } from "booru";
-import Post from "booru/dist/structures/Post"; // Ce n'est pas bien
-import { createReadStream, createWriteStream } from "fs";
-import { unlink } from "fs/promises";
-import got from "got-cjs";
-import stream from "node:stream";
-import { promisify } from "node:util";
-import { exit } from "process";
-import args from "./helpers/args.js";
-import crashHandler from "./helpers/crashHandler.js";
-import { config } from "./helpers/types.js";
-import postImage from "./postImage.js";
+import { createReadStream, createWriteStream } from "node:fs"
+import { unlink } from "node:fs/promises"
+import { exit } from "node:process"
+import stream from "node:stream"
+import { promisify } from "node:util"
+
+import { search, Post } from "booru"
+import got from "got-cjs"
+import args from "./helpers/args.js"
+import crashHandler from "./helpers/crashHandler.js"
+import { config } from "./helpers/types.js"
+import postImage from "./postImage.js"

 /**
 * Get a remote image from a booru
@ -23,43 +23,43 @@ export default async function getRemoteImage(conf: config) {
    limit: 1,
    random: true,
  }).catch((e) => {
-    crashHandler("Error searching for posts.", e);
-    return [] as Post[];
-  });
+    crashHandler("Error searching for posts.", e)
+    return [] as Post[]
+  })
  if (searchResults.length === 0) {
-    crashHandler("Error searching for posts.", Error("No posts found."));
-    return;
+    crashHandler("Error searching for posts.", Error("No posts found."))
+    return
  }
-  const post = searchResults[0];
-  if (args.verbose) console.log(`Found post: ${post.id} at ${post.file_url}`);
+  const post = searchResults[0]
+  if (args.verbose) console.log(`Found post: ${post.id} at ${post.file_url}`)
  // Set the post as sensitive if the rating is not safe
-  const sensitivity: boolean = post.rating !== "s";
+  const sensitivity: boolean = post.rating !== "s"
  // Make an HTTP request for the image
-  const filename: string = post.fileUrl?.split("/").pop() as string; // Type checks for type checks
-  const pipeline = promisify(stream.pipeline);
+  const filename: string = post.fileUrl?.split("/").pop() as string // Type checks for type checks
+  const pipeline = promisify(stream.pipeline)

  // Make the HTTP request as a stream so it can be piped to the file system
  await pipeline(
    got.stream(post.file_url as string),
    createWriteStream(filename)
  ).catch((err: Error) => {
-    crashHandler("Error saving downloading image.", err);
-  });
-  if (args.verbose) console.log(`Saved image to ${filename}`);
+    crashHandler("Error saving downloading image.", err)
+  })
+  if (args.verbose) console.log(`Saved image to ${filename}`)
  const str = createReadStream(filename).on("error", (err: Error) => {
-    crashHandler("Error reading downloaded image.", err);
-  });
+    crashHandler("Error reading downloaded image.", err)
+  })

  if (args.verbose) {
-    console.log(`File being sent: ${filename}\nSensitivity: ${sensitivity}`);
+    console.log(`File being sent: ${filename}\nSensitivity: ${sensitivity}`)
  }
  // Make a status with the image
-  await postImage(str, sensitivity, conf);
+  await postImage(str, sensitivity, conf)

  // Delete the image that it downloaded
  await unlink(filename).catch((err: Error) => {
-    crashHandler("Error deleting downloaded image.", err);
-  });
-  if (args.verbose) console.log(`Successfully deleted image ${filename}`);
-  exit(0);
+    crashHandler("Error deleting downloaded image.", err)
+  })
+  if (args.verbose) console.log(`Successfully deleted image ${filename}`)
+  exit(0)
 }
--- a/src/helpers/args.ts
+++ b/src/helpers/args.ts
@ -1,7 +1,8 @@
-import commandLineArgs from "command-line-args";
-import commandLineUsage from "command-line-usage";
-import { exit } from "process";
-import writeConfig from "./writeconfig.js";
+import { exit } from "node:process"
+
+import commandLineArgs from "command-line-args"
+import commandLineUsage from "command-line-usage"
+import writeConfig from "./writeconfig.js"

 const optionDefinitions = [
  {
@ -24,7 +25,7 @@ const optionDefinitions = [
    description:
      "Path to the JSON configuration file. (default: ./config.jsonc)",
    defaultValue: "./config.jsonc",
-    typeLabel: "<file.json[c]>",
+    typeLabel: "<file.json[c,5]>",
  },
  {
    name: "writeConfig",
@ -33,9 +34,9 @@ const optionDefinitions = [
    description:
      "Write a default configuration file to the current directory and exit.",
  },
-];
+]

-const args = commandLineArgs(optionDefinitions);
+const args = commandLineArgs(optionDefinitions)

 if (args.help) {
  const usage = commandLineUsage([
@ -52,17 +53,18 @@ if (args.help) {
      content:
        "Project home: {underline https://git.froth.zone/Sam/fediverse-imagebot}",
    },
-  ]);
-  console.log(usage);
-  exit(0);
+  ])
+  console.log(usage)
+  exit(0)
 }

-if (args.verbose) console.log("Running in verbose mode.\n");
+if (args.verbose) console.log("Running in verbose mode.\n")

 if (args.writeConfig) {
-  writeConfig(args.verbose);
-  console.log("Wrote default config file to ./config.jsonc");
-  exit(0);
+  writeConfig(args.verbose).then(() => {
+    console.log("Wrote default config file to ./config.jsonc")
+    exit(0)
+  })
 }

-export default args;
+export default args
--- a/src/helpers/crashHandler.ts
+++ b/src/helpers/crashHandler.ts
@ -1,5 +1,6 @@
-import { exit } from "process";
-import args from "./args.js";
+import { exit } from "node:process"
+
+import args from "./args.js"

 /**
 * The function that gets called when the program runs into an error.
@ -8,10 +9,10 @@ import args from "./args.js";
 * @returns This function will never return.
 */
 export default function crashHandler(msg: string, e: Error, res?: string) {
-  console.error(`${msg}: ${e.name}`);
+  console.error(`${msg}: ${e.name}`)
  if (args.verbose) {
-    console.error(`--BEGIN FULL ERROR--\n${e}\n${res}\n--END FULL ERROR--`);
-  } else console.error("Run with -v to see the full error.");
+    console.error(`--BEGIN FULL ERROR--\n${e}\n${res}\n--END FULL ERROR--`)
+  } else console.error("Run with -v to see the full error.")

-  exit(1);
+  exit(1)
 }
--- a/src/helpers/getConfig.ts
+++ b/src/helpers/getConfig.ts
@ -1,8 +1,9 @@
-import { readFile } from "fs/promises";
-import JSON5 from "json5";
-import args from "./args.js";
-import crashHandler from "./crashHandler.js";
-import { config } from "./types.js";
+import { readFile } from "node:fs/promises"
+
+import JSON5 from "json5"
+import args from "./args.js"
+import crashHandler from "./crashHandler.js"
+import { config } from "./types.js"

 /**
 * Reads the config file and returns the config object
@ -10,9 +11,9 @@ import { config } from "./types.js";
 */
 async function readConfig(): Promise<string> {
  return readFile(args.config, "utf8").catch((err) => {
-    crashHandler("Error reading config file.", err);
-    return "CRASH";
-  });
+    crashHandler("Error reading config file.", err)
+    return "CRASH"
+  })
 }
 /**
 * Parses the config file and returns it as a JSON object
@ -25,20 +26,23 @@ async function readConfig(): Promise<string> {
 * // Prints "https://mastodon.social"
 */
 export default async function getConfig(): Promise<config> {
-  let conf: config;
+  let conf: config
  try {
    conf = JSON5.parse(
      await readConfig().catch((err) => {
-        crashHandler("Error reading config file.", err);
-        return "";
+        crashHandler("Error reading config file.", err)
+        return ""
      })
-    );
+    )
+    // Backwards compatibility for older versions
+    conf.retries ??= 5
+    if (conf.retries < 1) conf.retries = 1
  } catch (err: unknown) {
-    crashHandler("Error parsing config file.", Error(err as string));
-    return {} as config;
+    crashHandler("Error parsing config file.", Error(err as string))
+    return {} as config
  }
  if (args.verbose) {
-    console.log(`Read config file: ${args.config}\n${JSON.stringify(conf)}`);
+    console.log(`Read config file: ${args.config}\n${JSON.stringify(conf)}`)
  }
-  return conf;
+  return conf
 }
--- a/src/helpers/types.ts
+++ b/src/helpers/types.ts
@ -2,14 +2,15 @@
 * Config type to reduce boilerplate
 */
 export type config = {
-  instance: string;
-  type: "misskey" | "mastodon" | "pleroma";
-  accessToken: string;
-  refreshToken: string | null;
-  message: string;
-  visibility: "direct" | "unlisted" | "private" | "public";
-  remote: boolean;
-  booru: string;
-  tags: string[];
-  directory: string;
-};
+  instance: string
+  type: "misskey" | "mastodon" | "pleroma"
+  accessToken: string
+  refreshToken: string | null
+  message: string
+  visibility: "direct" | "unlisted" | "private" | "public"
+  remote: boolean
+  booru: string
+  tags: string[]
+  directory: string
+  retries: number
+}
--- a/src/helpers/writeconfig.ts
+++ b/src/helpers/writeconfig.ts
@ -1,15 +1,14 @@
-import { writeFileSync } from "fs";
+import { writeFile } from "node:fs/promises"
 /**
 * Writes the sample config file to disk
 * @returns Nothing
 */
-export default function writeConfig(verbose: boolean) {
-  if (verbose) console.log("Writing sample config to config.jsonc");
-  writeFileSync(
+export default async function writeConfig(verbose: boolean) {
+  if (verbose) console.log("Writing sample config to config.jsonc")
+  await writeFile(
    "./config.jsonc",
    // eslint-disable-next-line prettier/prettier
    // prettier-ignore
    "{ \n  //Instance and token settings \n  \"instance\": \"INSTANCE_URL\", // example https://test.com \n  \"type\": \"INSTANCE_TYPE\", // examples: \"mastodon\", \"misskey\", \"pleroma\" \n  \"accessToken\": \"ACCESS_TOKEN\", // Get a token from https://git.froth.zone/Sam/js-feditoken \n  \"refreshToken\": \"REFRESH_TOKEN\", // optional \n \n  // Post settings \n  \"message\": \"\", // example: \"Hello, world!\" \n  \"visibility\": \"unlisted\", // example: \"public\", \"unlisted\", \"private\", \"direct\" \n \n  \"remote\": false, // **Set this to `true` if you want to serve a file from a booru!** \n  /* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `false` */ \n  \"booru\": \"safebooru.org\", // example: \"safebooru.org\" \n  \"tags\": [\"\"], // example: [\"tohsaka_rin\", \"-feet\"] \n  \"rating\": \"safe\", // example: \"safe\", \"questionable\", \"explicit\" \n  /* END OF SETTINGS THAT WILL BE IGNORED IF YOU SET `remote` TO `false` */ \n \n  /* THESE SETTINGS WILL BE IGNORED IF YOU SET `remote` TO `true` */ \n  \"directory\": \"./images\" // example: \"./images\" \n  /* \n    Directory structure should be as follows: \n    folder/ \n      - sfw/ \n        - image1.jpg \n      - nsfw/ \n        - image1.jpg \n  */ \n  /* END OF SETTINGS THAT WILL BE IGNORED IF YOU SET `remote` TO `true` */ \n} \n"
-  );
-  return;
+  )
 }
--- a/src/postImage.ts
+++ b/src/postImage.ts
@ -1,8 +1,10 @@
-import { ReadStream } from "fs";
-import generator, { Entity, Response } from "megalodon";
-import { Readable } from "stream";
-import crashHandler from "./helpers/crashHandler.js";
-import { config } from "./helpers/types.js";
+import { ReadStream } from "node:fs"
+import { Readable } from "node:stream"
+
+import generator, { Entity, Response } from "megalodon"
+import pRetry from "p-retry"
+import crashHandler from "./helpers/crashHandler.js"
+import { config } from "./helpers/types.js"

 /**
 * Uploads an image to a fediverse instance
@ -22,26 +24,37 @@ export default async function postImage(
    cfg.instance,
    cfg.accessToken,
    cfg.refreshToken
-  );
+  )

  // Upload the image
-  const res: Response<Entity.Attachment> = await client
-    .uploadMedia(image)
-    .catch((err) => {
-      crashHandler("Error uploading image.", err, err.response.data);
-      return {} as Response<Entity.Attachment>;
-    });
+  const upload = async () => client.uploadMedia(image)
+  const res = await pRetry(upload, {
+    retries: cfg.retries,
+    onFailedAttempt: logRetry,
+  }).catch((err) => {
+    crashHandler("Error uploading image.", err, err.response.data)
+    return {} as Response<Entity.Attachment>
+  })

-  // Make a status with the image
-  await client
-    .postStatus(cfg.message, {
+  // Make the post
+  const post = async () =>
+    client.postStatus(cfg.message, {
      media_ids: [res.data.id],
      // Change this to make the post visibility you wish
      visibility: cfg.visibility,
      sensitive: sensitivity,
    })
-    .catch((err: Error) => {
-      crashHandler("Error posting status.", err);
-    });
-  console.log(`Successfully posted to ${cfg.instance}`);
+  await pRetry(post, {
+    retries: cfg.retries,
+    onFailedAttempt: logRetry,
+  }).catch((err) => {
+    crashHandler("Error uploading image", err, err.response.data)
+  })
+
+  console.log(`Successfully posted to ${cfg.instance}`)
+}
+
+function logRetry(error: unknown) {
+  console.error("Retrying, error:")
+  console.error(error)
 }
--- a/tsconfig.json
+++ b/tsconfig.json
@ -2,7 +2,7 @@
  "compilerOptions": {
    /* Basic Options */
    "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */,
-    "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
+    "module": "CommonJS" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
    "lib": [
      "es6"
    ] /* Specify library files to be included in the compilation. */,