Initial commit

.dockerignore

@@ -0,0 +1 @@
+**/node_modules

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+insert_final_newline = true

.env.sample

@@ -0,0 +1 @@
+PORT=3000

.eslintignore

@@ -0,0 +1 @@
+build/

.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "./node_modules/gts/"
+}

.gitignore

@@ -128,3 +128,6 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+data/
+.idea/

.prettierrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('gts/.prettierrc.json')
+}

Dockerfile

@@ -0,0 +1,15 @@
+FROM node:18
+
+#install calibre
+RUN apt-get update -y
+RUN apt-get install -y libopengl0
+RUN apt-get install -y libegl1
+RUN wget -nv -O- https://download.calibre-ebook.com/linux-installer.sh | sh /dev/stdin
+
+WORKDIR /srv/app
+COPY ./ /srv/app
+RUN npm install
+
+EXPOSE $PORT
+
+CMD ["npm", "run", "dev"]

README.md

@@ -1 +1,16 @@
 # kobo
+work in progress.
+
+it is intended for 2 things:
+1. web articles in epub format. (sort of implemented)
+   1. using your pc's (or phone's) web browser, add article's url to this web server.
+   2. it will download the article (and images) and convert it to epub
+   3. you can download the epub file using kobo device.
+2. search and download your epubs from kobo device's web browser. (not implemented yet)
+
+### why?
+- pocket is great and natively supported by kobo, but it doesn't save images (at least not for the tried articles).
+- wallabag is also great, but it doesn't have a native kobo app so you need koreader.
+
+### demo
+[https://kobo.sklk.lt](https://kobo.sklk.lt)

docker-compose.yaml

@@ -0,0 +1,12 @@
+services:
+  server:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    volumes:
+      - .:/srv/app/
+      - ./data:/data
+    env_file:
+      - .env
+    ports:
+      - $PORT:$PORT

package.json

@@ -0,0 +1,27 @@
+{
+  "devDependencies": {
+    "@types/express": "^4.17.17",
+    "@types/node": "^20.4.5",
+    "@types/sanitize-html": "^2.9.0",
+    "gts": "^5.0.0",
+    "nodemon": "^3.0.1",
+    "typescript": "~5.1.6"
+  },
+  "scripts": {
+    "lint": "gts lint",
+    "clean": "gts clean",
+    "compile": "tsc",
+    "fix": "gts fix",
+    "prepare": "npm run compile",
+    "pretest": "npm run compile",
+    "posttest": "npm run lint",
+    "dev": "nodemon --watch 'src/**/*.ts' --exec 'ts-node' src/index.ts"
+  },
+  "dependencies": {
+    "@extractus/article-extractor": "^7.3.1",
+    "cheerio": "^1.0.0-rc.12",
+    "express": "^4.18.2",
+    "express-handlebars": "^7.1.2",
+    "ts-node": "^10.9.1"
+  }
+}

src/helpers/generate-pagination.ts

@@ -0,0 +1,18 @@
+import {RelativeUrl} from "./relative-url.helper";
+
+export const generatePagination = (page: number, pages: number, url: RelativeUrl) => {
+  const prev = page > 1 ? url.setSearchParam('page', (page - 1).toString()).toString() : '';
+  const next = page < pages ? url.setSearchParam('page', (page + 1).toString()).toString()  : '';
+  const pagination = [prev];
+
+  for (let i = 1; i <= pages; i++) {
+    if (page === i) {
+      pagination.push('');
+    } else {
+      pagination.push(url.setSearchParam('page', i.toString()).toString());
+    }
+  }
+  pagination.push(next);
+
+  return pagination;
+}

src/helpers/relative-url.helper.ts

@@ -0,0 +1,16 @@
+export class RelativeUrl {
+  private readonly url: URL;
+  constructor(input: string) {
+    this.url = new URL(input, 'relative:///');
+  }
+
+  toString(): string {
+    return this.url.toString().replace('relative://', '');
+  }
+
+  setSearchParam(key: string, value: string): this {
+    this.url.searchParams.set(key, value);
+
+    return this;
+  }
+}

src/index.ts

@@ -0,0 +1,37 @@
+import * as express from 'express';
+import {engine} from "express-handlebars";
+import {indexRouter} from "./routes";
+import {searchRouter} from "./routes/search";
+import {join} from "path";
+import {addArticleRouter} from "./routes/add-article";
+import {articlesRouter} from "./routes/articles";
+
+const PORT = process.env.PORT || 3000;
+const app = express();
+
+const hbsOptions = {
+  extname: '.hbs',
+  helpers: {
+    toJSON: (obj: any) => JSON.stringify(obj, null, 2),
+  }
+}
+
+app.engine('.hbs', engine(hbsOptions));
+app.set('view engine', '.hbs');
+app.set('views', 'src/views');
+
+app.use(express.static(join(__dirname, 'public')));
+
+app.use("*", (req, res, next) => {
+  console.log(req.originalUrl);
+  next();
+});
+
+app.use("/", indexRouter);
+app.use("/search", searchRouter);
+app.use("/add-article", addArticleRouter);
+app.use("/articles", articlesRouter);
+
+app.listen(PORT, () => {
+    console.log(`Server is running on port ${PORT}`);
+});

src/public/style.css

@@ -0,0 +1,39 @@
+.col-7 {
+  width: 70%;
+}
+
+.kobo-input {
+  font-size: xx-large;
+  border: 4px black solid;
+}
+
+.pagination {
+  display: inline-block;
+}
+
+.pagination span {
+  color: black;
+  float: left;
+  padding: 10px 20px;
+  margin: 4px;
+  text-decoration: none;
+  border: 1px black solid;
+  font-size: x-large;
+}
+
+a {
+  text-decoration: none;
+  color: black;
+}
+
+a div {
+  background-color: buttonface;
+  font-size: x-large;
+}
+
+div .item {
+  font-size: x-large;
+  border: black 2px solid;
+  margin-bottom: 20px;
+  padding: 10px;
+}

src/repository/articles-repository.ts

@@ -0,0 +1,67 @@
+import {existsSync, readFileSync} from "fs";
+import {writeFile} from "fs/promises";
+
+interface Article {
+  id: number;
+  title: string;
+  dateAdded: string;
+  uid: string;
+}
+
+class ArticlesRepository {
+  private readonly articles: Article[] = [];
+  private counter: number = 0;
+
+  constructor() {
+    if (!existsSync('/data/articles.json')) {
+      return;
+    }
+    const rawData = readFileSync('/data/articles.json', 'utf-8');
+    const data = JSON.parse(rawData);
+    this.articles = data.articles;
+    this.counter = data.counter;
+  }
+
+  public add(article: Omit<Article, "id">): Article {
+    const savedArticle = this.getByUid(article.uid);
+    if (savedArticle) {
+      //push article to the top
+      this.articles.splice(this.articles.indexOf(savedArticle), 1);
+      this.articles.unshift(savedArticle);
+
+      Object.assign(savedArticle, article);
+      this.saveDb();
+      return savedArticle;
+    }
+
+    const completeArticle: Article = {
+      ...article,
+      id: this.counter++,
+    }
+    this.articles.unshift(completeArticle);
+    this.saveDb();
+
+    return completeArticle;
+  }
+
+  public getAll(): Article[] {
+    return this.articles;
+  }
+
+  public get(id: number): Article | undefined {
+    return this.articles.find((article) => article.id === id);
+  }
+
+  public getByUid(uid: string): Article | undefined {
+    return this.articles.find((article) => article.uid === uid);
+  }
+
+  private async saveDb(): Promise<void> {
+    await writeFile('/data/articles.json', JSON.stringify({
+      articles: this.articles,
+      counter: this.counter,
+    }, null, 2));
+  }
+}
+
+export const articlesRepository = new ArticlesRepository();

src/routes/add-article.ts

@@ -0,0 +1,152 @@
+import * as express from 'express';
+import {extract} from '@extractus/article-extractor'
+import {createWriteStream, existsSync, mkdir, writeFile} from "fs";
+import * as crypto from "crypto";
+import * as cheerio from 'cheerio';
+import {Readable} from "stream";
+import {rm} from "fs/promises";
+import {articlesRepository} from "../repository/articles-repository";
+import {ReadableStream} from "node:stream/web";
+import {spawn} from "child_process";
+
+export const addArticleRouter = express.Router();
+/*
+ebook-convert article.html article.epub --title="Bank of England signals ‘significant’ response to pound turmoil" --authors="Article"
+*/
+
+addArticleRouter.get('/', (req, res) => {
+  const url = req.query.url as string;
+
+  if (url) {
+    extract(url as string).then((article) => {
+      const hash = crypto.createHash('md5').update(url).digest('hex');
+      const dir = `/data/articles/${hash}`;
+      if (!article) {
+        res.render('article-added', {
+          title: 'article not found',
+        });
+        return;
+      }
+
+      const articleEntity = articlesRepository.add({
+        uid: hash,
+        title: article.title ?? 'no title',
+        dateAdded: new Date().toISOString().split('T')[0],
+      });
+
+      mkdir(dir, {recursive: true}, (err) => {
+        if (err) throw err;
+        writeFile(`${dir}/article.json`, JSON.stringify(article, null, 2), (err) => {
+          if (err) throw err;
+          console.log('json file saved');
+          const rawHtml = article.content as string;
+          const $ = cheerio.load(rawHtml);
+
+          const downloadImagesPromises: Promise<void>[] = [];
+
+          $('img').each((i, el) => {
+            const src = $(el).attr('src');
+            if (src) {
+              const img = src.split('/').pop();
+              const imgPath = `${dir}/${img}`;
+
+              downloadImagesPromises.push(downloadImage(src, imgPath));
+
+              $(el).attr('src', `./${img}`);
+            }
+          });
+
+          Promise.all<void>(downloadImagesPromises).then(() => {
+            console.log('all images downloaded');
+            let commandOutput = '';
+            const command = spawn('ebook-convert', [
+              `${dir}/article.html`,
+              `${dir}/article.epub`,
+              '--title',
+              article.title ?? 'no title',
+              '--authors',
+              'Article',
+            ]);
+            command.on('error', (err) => {
+              console.log(err);
+            });
+            command.stdout.on('data', (data) => {
+              commandOutput += data.toString();
+            });
+            command.stderr.on('data', (data) => {
+              commandOutput += data.toString();
+            })
+            command.on('close', (code) => {
+              writeFile(`${dir}/article-convert-output.txt`, commandOutput, (err) => {});
+              console.log(`book conversion ended with ${code}`);
+              res.render('article-added', {
+                title: 'article added',
+                id: articleEntity.id,
+              });
+            });
+          })
+
+          writeFile(`${dir}/article.html`, $.html(), (err) => {
+            console.log('html file saved');
+          });
+
+        });
+      });
+    })
+    return;
+  }
+
+  res.render('add-article', {
+    title: "please article enter url",
+  });
+});
+
+function downloadImage(imgSrc: string, imgPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (existsSync(imgPath)) {
+      //don't download if already exists
+      console.log(`${imgPath} already exists`);
+      return resolve();
+    }
+    fetch(imgSrc).then((res) => {
+      const body = res.body;
+      if (body === null) {
+        console.log(`${imgSrc} not downloadable`);
+
+        return;
+      }
+
+      // @ts-ignore
+      downloadStream(imgPath, body)
+        .then(() => resolve())
+        .catch((e) => reject(e));
+    })
+  });
+}
+
+function deleteIfExists(path: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (existsSync(path)) {
+      return rm(path).then(() => resolve()).catch((e) => reject(e));
+    }
+
+    return resolve();
+  });
+}
+
+function downloadStream(path: string, readableStream: ReadableStream): Promise<void> {
+  return new Promise((resolve, reject) => {
+    deleteIfExists(path).then(() => {
+      const dest = createWriteStream(path, {flags: 'wx'});
+      Readable.fromWeb(readableStream)
+        .pipe(dest)
+        .on("finish", () => {
+          console.log(`${path} downloaded`);
+          resolve();
+        })
+        .on("error", (err) => {
+          reject(err);
+        });
+    }).catch((e) => reject(e));
+  });
+}

src/routes/articles.ts

@@ -0,0 +1,31 @@
+import * as express from 'express';
+import {RelativeUrl} from "../helpers/relative-url.helper";
+import {articlesRepository} from "../repository/articles-repository";
+import {generatePagination} from "../helpers/generate-pagination";
+
+export const articlesRouter = express.Router();
+
+articlesRouter.get('/:id', (req, res) => {
+  const article = articlesRepository.get(Number(req.params.id));
+  if (article) {
+    return res.download(`/data/articles/${article.uid}/article.epub`);
+  }
+  throw new Error('article not found');
+});
+
+articlesRouter.get('/', (req, res) => {
+  const page: number = parseInt(req.query.page as string) || 1;
+  const perPage = 6;
+  const articles = articlesRepository.getAll();
+  const searchResults = articles.slice((page - 1) * perPage, page * perPage);
+  const pages = Math.ceil(articles.length / perPage);
+
+  const url = new RelativeUrl(req.originalUrl);
+
+  res.render('articles', {
+    title: `articles`,
+    searchResults,
+    pagesList: generatePagination(page, pages, url),
+  });
+});
+

src/routes/index.ts

@@ -0,0 +1,8 @@
+import * as express from 'express';
+
+export const indexRouter = express.Router();
+
+indexRouter.get('/', (req, res) => {
+  res.render('index', { title: "main"} );
+});
+

src/routes/search.ts

@@ -0,0 +1,37 @@
+import * as express from 'express';
+import {RelativeUrl} from "../helpers/relative-url.helper";
+import {generatePagination} from "../helpers/generate-pagination";
+
+export const searchRouter = express.Router();
+
+const searchQueries = [
+  "1. first entry",
+  "2. second entry",
+  "3. third entry",
+  "4. fourth entry",
+  "5. fifth entry",
+  "6. sixth entry",
+  "7. seventh entry",
+  "8. eighth entry",
+  "9. ninth entry",
+  "10. tenth entry",
+  "11. eleventh entry",
+];
+
+searchRouter.get('/', (req, res) => {
+  const searchQuery = req.query.query;
+
+  const page: number = parseInt(req.query.page as string) || 1;
+  const perPage = 6;
+  const searchResults = searchQueries.slice((page - 1) * perPage, page * perPage);
+  const pages = Math.ceil(searchQueries.length / perPage);
+
+  const url = new RelativeUrl(req.originalUrl);
+
+  res.render('search', {
+    title: `search results for ${searchQuery}`,
+    searchResults,
+    query: searchQuery,
+    pagesList: generatePagination(page, pages, url),
+  });
+});

src/views/add-article.hbs

@@ -0,0 +1,13 @@
+<div>
+  <a href="/"><div class="item">Back to main</div></a>
+</div>
+<div>
+  <form method="get" action="add-article">
+    <div style="margin: 10px;">
+      <label>url
+        <input type="text" name="url" value="{{url}}" class="col-7" style="font-size: large; width: 90%">
+      </label>
+        <input type="submit" value="ok" style="font-size: large"/>
+    </div>
+  </form>
+</div>

src/views/article-added.hbs

@@ -0,0 +1,14 @@
+<div>
+  <a href="/"><div class="item">Back to main</div></a>
+</div>
+<div>
+  <a href="/add-article"><div class="item">Add another</div></a>
+</div>
+<div>
+  <a href="/articles"><div class="item">Articles</div></a>
+</div>
+{{#if id}}
+  <div>
+    <a href="/articles/{{id}}"><div class="item">Download</div></a>
+  </div>
+{{/if}}

src/views/articles.hbs

@@ -0,0 +1,12 @@
+<a href="/">
+  <div class="item">Back to main</div>
+</a>
+<div>
+  {{#each searchResults}}
+    <a href="/articles/{{this.id}}">
+      <div class="item">
+        {{this.title}}
+      </div>
+    </a>
+  {{/each}}
+</div>

src/views/index.hbs

@@ -0,0 +1,20 @@
+<div>
+  <h2>books</h2>
+  <form method="get" action="search">
+    <input class="kobo-input" type="text" name="query" value="{{query}}" class="col-7" tabindex="-1">
+    <input class="kobo-input" type="submit" value="search"/>
+  </form>
+  <a href="books">
+    <div class="item">Browse all</div>
+  </a>
+</div>
+
+<div>
+  <h2>articles</h2>
+  <a href="articles">
+    <div class="item">Articles</div>
+  </a>
+  <a href="add-article">
+    <div class="item">Add article</div>
+  </a>
+</div>

src/views/layouts/main.hbs

@@ -0,0 +1,32 @@
+<html lang="lt">
+<head>
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>kobo</title>
+  <link rel="stylesheet" href="style.css"/>
+</head>
+
+<body>
+<h1>
+  {{title}}
+</h1>
+
+<div>
+  {{{body}}}
+</div>
+
+<div>
+  {{#if pagesList}}
+    {{> pagination }}
+  {{/if}}
+</div>
+
+<div>
+  {{#if debug}}
+    <pre>{{toJSON debug}}
+    </pre>
+  {{/if}}
+</div>
+
+</body>
+</html>
+

src/views/partials/pagination.hbs

@@ -0,0 +1,25 @@
+<div class="pagination">
+  {{#each pagesList}}
+    {{#if @first}}
+      {{#if this}}
+        <a href="{{this}}"><span>&laquo;</span></a>
+      {{else}}
+        <span style="color: grey">&laquo;</span>
+      {{/if}}
+    {{else}}
+      {{#if @last}}
+        {{#if this}}
+          <a href="{{this}}"><span>&raquo;</span></a>
+        {{else}}
+          <span>&raquo;</span>
+        {{/if}}
+      {{else}}
+        {{#if this}}
+          <a href="{{this}}"><span>{{@index}}</span></a>
+        {{else}}
+          <span style="background-color: black; color: white">{{@index}}</span>
+        {{/if}}
+      {{/if}}
+    {{/if}}
+  {{/each}}
+</div>

src/views/search.hbs

@@ -0,0 +1,9 @@
+
+  <a href="/"><div class="item">Back to main</div></a>
+<div>
+    {{#each searchResults}}
+      <div class="item">
+      {{this}}
+      </div>
+    {{/each}}
+</div>

tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "./node_modules/gts/tsconfig-google.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "outDir": "build",
+    "lib": ["dom"]
+  },
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts"
+  ]
+}