multi downloads

This commit is contained in:
Daniel Bulant 2021-05-25 15:44:10 +02:00
parent e13ac35fea
commit f94398cef8
7 changed files with 479 additions and 208 deletions

View file

@ -0,0 +1,54 @@
<script>
import { url } from "@roxi/routify/runtime/helpers";
import { createEventDispatcher } from "svelte";
const dispatch = createEventDispatcher();
export var chapter;
export var selected;
export var disabledDownload = false;
</script>
<tr on:click={() => dispatch("select")} class:selected={selected}>
<td class="no-wrap">{chapter.data.attributes.volume ? "Vol " + chapter.data.attributes.volume : ""}</td>
<td class="no-wrap">Chapter {chapter.data.attributes.chapter}</td>
<td>{chapter.data.attributes.title}</td>
<td class="action no-wrap"><span on:click|stopPropagation={() => !disabledDownload && dispatch("download")} class:disabled={disabledDownload}>Download</span></td>
<td class="action no-wrap"><a href={$url("./" + chapter.data.id)} on:click|stopPropagation={() => !disabledDownload && dispatch("view")}>View</a></td>
</tr>
<style>
tr {
border: 1px solid black;
}
tr.selected {
background: rgba(0,0,0,0.15);
}
tr:hover {
background: rgba(0,0,0,0.2);
}
tr.selected:hover {
background: rgba(0,0,0,0.25);
}
td {
padding: 5px 5px;
}
td.action {
font-weight: bold;
color: black;
text-decoration: none;
cursor: pointer;
user-select: none;
}
td.action:hover {
text-decoration: underline;
color: rgb(0,100,200);
}
td.action a {
color: inherit;
}
</style>

View file

@ -1,7 +1,8 @@
<script>
import { page, url } from "@roxi/routify/runtime/helpers";
import { Zip, ZipPassThrough } from "fflate";
import { prepareEpub } from "../../util/generateEpub";
import { url } from "@roxi/routify/runtime/helpers";
import Chapter from "../../components/chapter.svelte";
import { EpubGenerator } from "../../util/generateEpub";
import { CBZGenerator } from "../../util/generateCbz";
// import * as streamSaver from "streamsaver";
import request from "../../util/request";
@ -29,84 +30,130 @@
var progress = 0;
var state = "idle";
var text = "Choose a chapter to view online or download EPUB";
const defaultText = "Choose a chapter to view online or download EPUB";
var text = defaultText;
var pagesDone = 0;
var totalPages = 0;
$: progress = pagesDone / (totalPages || 1);
$: if(totalPages) text = `Saving page ${pagesDone + 1} of ${totalPages}`;
var enc = new TextEncoder();
async function prepare(chapter) {
state = "active";
text = "Starting download of chapter " + chapter.data.attributes.chapter;
/**
* @type {EpubGenerator[]}
*/
var queue = [];
var processing = null;
async function processQueue() {
if(processing) return;
processing = queue.shift();
if(!processing) return processing = null;
processing.opts.callback = (chapter, link, finished) => {
console.log(chapter, link, finished);
};
await processing.generate();
processing = null;
processQueue();
}
const { baseUrl } = await request("at-home/server/" + chapter.data.id);
const quality = "data";
const generators = {
epub: EpubGenerator,
cbz: CBZGenerator
}
const quality = "data";
const URLs = [];
const hashes = [];
for(const hash of chapter.data.attributes[quality]) {
URLs.push(`${baseUrl}/${quality}/${chapter.data.attributes.hash}/${hash}`);
hashes.push(hash);
}
text = "Found " + URLs.length + " pages";
totalPages += URLs.length;
const file = streamSaver.createWriteStream(`${manga.title.en} ${chapter.data.attributes.chapter}.epub`, {
async function downloadSingle(chapter) {
const file = streamSaver.createWriteStream(`${manga.title.en} ${chapter.data.attributes.chapter}.${format}`, {
writableStrategy: undefined, // (optional)
readableStrategy: undefined, // (optional)
});
const zip = await prepareEpub({
title: `${manga.title.en} ${chapter.data.attributes.chapter}`,
id: `https://manga.danbulant.eu/${mangaId}/${chapter.data.id}`,
const generator = new generators[format]({
file,
chapter: chapter.data.attributes.chapter,
links: hashes,
updatedAt: chapter.data.attributes.updatedAt
id: chapter.data.id,
language: chapter.data.attributes.translatedLanguage,
quality,
updatedAt: chapter.data.attributes.updatedAt,
title: manga.title.en,
author: "Unknown",
chapters: [{
hash: chapter.data.attributes.hash,
id: chapter.data.id,
links: chapter.data.attributes[quality],
number: chapter.data.attributes.chapter,
volume: chapter.data.attributes.volume
}]
});
for(var i = 0; i < URLs.length; i++) {
const url = URLs[i];
const hash = hashes[i];
const res = await fetch(url);
const image = new ZipPassThrough("OEBPS/" + hash);
zip.add(image);
image.push(new Uint8Array(await res.arrayBuffer()), true);
const textContent = new ZipPassThrough("OEBPS/" + i + ".xhtml");
zip.add(textContent);
textContent.push(enc.encode(`<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<title>Page ${i + 1}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="EPB-UUID" content=""/>
</head>
<body>
<img style="margin:auto;height:100%;" src="${hash}" />
</body>
</html>`), true);
pagesDone++;
}
zip.end();
if(pagesDone === totalPages) {
text = "Done!";
state = "idle";
pagesDone = 0;
totalPages = 0;
console.log(generator);
queue.push(generator);
processQueue();
}
setTimeout(() => {
if(totalPages === 0) {
text = "Choose a chapter to view online or download EPUB";
}
}, 3000);
var format = "cbz";
var selected = [];
function select(chapter) {
if(selected.includes(chapter)) {
selected.splice(selected.indexOf(chapter), 1);
} else {
selected.push(chapter);
}
selected = selected;
if(selected.length) {
text = `Selected ${selected.length} chapters`;
} else {
text = defaultText;
}
}
function downloadMulti() {
selected.sort((a, b) => a.data.attributes.chapter - b.data.attributes.chapter);
if(!selected.length) return;
if(selected.length === 1) {
downloadSingle(selected.shift());
selected = [];
return;
}
const file = streamSaver.createWriteStream(`${manga.title.en}.${format}`, {
writableStrategy: undefined, // (optional)
readableStrategy: undefined, // (optional)
});
const generator = new generators[format]({
file,
quality,
id: window.location.toString(),
language: selected[0].data.attributes.translatedLanguage,
updatedAt: new Date,
title: manga.title.en,
author: "Unknown",
chapters: selected.map(chapter => ({
hash: chapter.data.attributes.hash,
id: chapter.data.id,
links: chapter.data.attributes[quality],
number: chapter.data.attributes.chapter,
volume: chapter.data.attributes.volume
}))
});
console.log(generator);
queue.push(generator);
selected = [];
processQueue();
}
/**
* @param {BeforeUnloadEvent} e
*/
function beforeUnload(e) {
if(progress) {
e.preventDefault();
return "Downloads won't be saved if you exit this page before they're finished.";
}
}
$: console.log(format);
</script>
<svelte:window on:beforeUnload={beforeUnload} />
<svelte:head>
<title>Chapters of {manga.title.en}</title>
</svelte:head>
@ -128,6 +175,18 @@
</p>
</div>
{#if queue.length > 0}
<p><i>{queue.length} downloads queued.</i></p>
{/if}
<div class="download">
<select name="format" bind:value={format} id="select-format">
<option value="cbz"><b>.cbz</b> Comic Book Zip</option>
<option value="epub"><b>.epub</b> Electronic publication</option>
</select>
<button disabled={!!progress && selected.length} on:click={downloadMulti}>Download</button>
</div>
<p>
<b>
Do not close the tab when a download is in progress.
@ -143,13 +202,7 @@
<table>
<tbody>
{#each chapters.results.filter(c => c.data.attributes.translatedLanguage === "en") as chapter}
<tr>
<td class="no-wrap">{chapter.data.attributes.volume ? "Vol " + chapter.data.attributes.volume : ""}</td>
<td class="no-wrap">Chapter {chapter.data.attributes.chapter}</td>
<td>{chapter.data.attributes.title}</td>
<td class="action no-wrap"><span on:click={() => prepare(chapter)}>Download</span></td>
<td class="action no-wrap"><a href={$url("./" + chapter.data.id)} on:click|stopPropagation>View</a></td>
</tr>
<Chapter {chapter} disabledDownload={!!progress} selected={selected.includes(chapter)} on:select={() => select(chapter)} on:download={() => downloadSingle(chapter)} />
{/each}
</tbody>
</table>
@ -157,6 +210,18 @@
</main>
<style>
.download {
display: flex;
width: 100%;
margin-top: 5px;
}
.download select {
flex-grow: 1;
margin-inline: 5px;
}
.download button {
margin-inline: 5px;
}
main {
font-size: 1.1rem;
}
@ -171,34 +236,6 @@
width: 100%;
}
tr {
border: 1px solid black;
}
tr:hover {
background: rgba(0,0,0,0.1);
}
td {
padding: 5px 5px;
}
td.action {
font-weight: bold;
color: black;
text-decoration: none;
cursor: pointer;
}
td.action:hover {
text-decoration: underline;
color: rgb(0,100,200);
}
td.action a {
color: inherit;
}
.state {
border-radius: 10px;
border-width: 4px;

73
src/util/baseGenerator.js Normal file
View file

@ -0,0 +1,73 @@
/**
* @typedef Chapter
* @property {string} id
* @property {number} number
* @property {number?} volume
* @property {string[]} links
* @property {string} hash
*/
import request, { proxy } from "./request";
/**
* Base generator, to be extended
*/
export class BaseGenerator {
/**
*
* @param {object} opts
* @param {string} opts.baseUrl
* @param {string} opts.quality
* @param {WritableStream} opts.file
* @param {string} opts.title
* @param {string} opts.id
* @param {string} opts.language
* @param {string} opts.author
* @param {string|Date} opts.updatedAt
* @param {Chapter[]} opts.chapters
* @param {(chapter: number, link: number, finished: boolean) => void} [opts.callback]
* @param {(error: Error) => void} [opts.onerror]
*/
constructor(opts) {
this.opts = opts;
}
async generate() {}
/**
* @param {string | Chapter | any} chapter
* @returns {Promise<string>}
*/
async getBaseURL(chapter) {
if(typeof chapter === "object") chapter = chapter.data.id;
const { baseUrl } = await request("at-home/server/" + chapter);
return baseUrl;
}
/**
* @param {string} url
* @returns {Promise<Response>}
*/
async fetchImage(url) {
var res;
try {
res = await fetch(url);
} catch(e) {
console.error(e);
res = await fetch(proxy + url);
}
return res;
}
/**
* @param {number} chapter
* @param {number} link
* @param {boolean} finished
*/
callback(chapter = -1, link = -1, finished = false) {
if(this.opts.callback) {
this.opts.callback(chapter, link, finished);
}
}
}

52
src/util/generateCbz.js Normal file
View file

@ -0,0 +1,52 @@
import { Zip, ZipPassThrough } from "fflate";
import { BaseGenerator } from "./baseGenerator";
import report from "./report";
export class CBZGenerator extends BaseGenerator {
async generate() {
this.writer = this.opts.file.getWriter();
this.zip = new Zip();
this.zip.ondata = (error, data, final) => {
if(error) {
console.error(error);
if(this.opts.onerror) this.opts.onerror(error);
}
if(data) {
this.writer.write(data);
}
if(final) {
this.writer.close();
}
};
this.hashes = this.opts.chapters.map(t => t.links).flat();
const chapterCountLength = this.opts.chapters.reduce((a, b) => Math.max(a.number, b.number)).number.toString().length;
for(const chapterI in this.opts.chapters) {
const chapter = this.opts.chapters[chapterI];
const baseUrl = await this.getBaseURL(chapter.id);
const imageCountLength = chapter.links.length.toString().length;
for(const i in chapter.links) {
this.callback(chapterI, i, false);
const hash = chapter.links[i];
const URL = `${baseUrl}/${this.opts.quality}/${chapter.hash}/${hash}`;
const start = performance.now();
const res = await this.fetchImage(URL);
const image = new ZipPassThrough(`${this.opts.title} ${chapter.number.toString().padStart(chapterCountLength, "0")}/${i.toString().padStart(imageCountLength, "0")}.${hash.substr(hash.lastIndexOf(".") + 1)}`);
this.zip.add(image);
const data = new Uint8Array(await res.arrayBuffer());
const end = performance.now() - start;
report({
bytes: data.byteLength,
cached: res.headers.get("X-Cache") === "HIT",
duration: end,
success: Math.floor(res.status / 100) === 2,
url: URL
});
image.push(data, true);
}
}
this.zip.end();
}
}

View file

@ -1,131 +1,161 @@
import { Zip, ZipPassThrough } from "fflate";
import { BaseGenerator } from "./baseGenerator";
import report from "./report";
const enc = new TextEncoder();
/**
* @param {object} opts
* @param {string[]} opts.links
* @param {WritableStream} opts.file
* @param {number} opts.chapter
* @param {string|Date} opts.updatedAt
* @param {string} opts.id
* Handles epub generation
*/
export async function prepareEpub(opts) {
const writer = opts.file.getWriter();
const zip = new Zip();
zip.ondata = (error, data, final) => {
if(error) {
console.error(error);
text = "Error: " + error.message;
state = "error";
export class EpubGenerator extends BaseGenerator {
async generate() {
this.writer = this.opts.file.getWriter();
this.zip = new Zip();
this.zip.ondata = (error, data, final) => {
if(error) {
console.error(error);
if(this.opts.onerror) this.opts.onerror(error);
}
if(data) {
this.writer.write(data);
}
if(final) {
this.writer.close();
}
};
this.hashes = this.opts.chapters.map(t => t.links).flat();
this.mimetype();
this.container();
this.package();
this.toc();
this.callback(); // signals the template is ready
for(const chapterI in this.opts.chapters) {
const chapter = this.opts.chapters[chapterI];
const baseUrl = await this.getBaseURL(chapter.id);
for(const i in chapter.links) {
this.callback(chapterI, i, false);
const hash = chapter.links[i];
const URL = `${baseUrl}/${this.opts.quality}/${chapter.hash}/${hash}`;
const start = performance.now();
const res = await this.fetchImage(URL);
const image = new ZipPassThrough("OEBPS/" + hash);
this.zip.add(image);
const data = new Uint8Array(await res.arrayBuffer());
const end = performance.now() - start;
report({
bytes: data.byteLength,
cached: res.headers.get("X-Cache") === "HIT",
duration: end,
success: Math.floor(res.status / 100) === 2,
url: URL
});
image.push(data, true);
const textContent = new ZipPassThrough("OEBPS/" + i + ".xhtml");
this.zip.add(textContent);
textContent.push(enc.encode(`<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<title>Page ${i + 1}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="EPB-UUID" content=""/>
</head>
<body>
<img style="margin:auto;height:100%;" src="${hash}" />
</body>
</html>`), true);
this.callback(chapterI, i, true);
}
}
if(data) {
writer.write(data);
}
if(final) {
writer.close();
}
};
const hashes = opts.links;
const mimetype = new ZipPassThrough("mimetype");
zip.add(mimetype);
mimetype.push(enc.encode("application/epub+zip"), true);
this.zip.end();
this.callback(-1, -1, true);
}
const container = new ZipPassThrough("META-INF/container.xml");
zip.add(container);
container.push(enc.encode(`<?xml version="1.0" encoding="UTF-8" ?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>`), true);
mimetype() {
const mimetype = new ZipPassThrough("mimetype");
this.zip.add(mimetype);
mimetype.push(enc.encode("application/epub+zip"), true);
}
const opf = new ZipPassThrough("OEBPS/content.opf");
zip.add(opf);
opf.push(enc.encode(`<?xml version="1.0"?>
<package version="3.0" xmlns="http://www.idpf.org/2007/opf" unique-identifier="bookid">
container() {
const container = new ZipPassThrough("META-INF/container.xml");
this.zip.add(container);
container.push(enc.encode(`<?xml version="1.0" encoding="UTF-8" ?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>`), true);
}
<metadata xmlns:opf="http://www.idpf.org/2007/opf" xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/">
<dc:title>${opts.title}</dc:title>
<dc:language>en</dc:language>
<dc:creator>Unknown</dc:creator>
<dc:identifier id="bookid">${opts.id}</dc:identifier>
<dc:type>Image</dc:type>
package() {
const opf = new ZipPassThrough("OEBPS/content.opf");
this.zip.add(opf);
opf.push(enc.encode(`<?xml version="1.0"?>
<package version="3.0" xmlns="http://www.idpf.org/2007/opf" unique-identifier="bookid">
<meta property="dcterms:modified">${opts.updatedAt.toString().split("+")[0]}Z</meta>
<meta property="rendition:layout">pre-paginated</meta>
<meta property="rendition:orientation">portrait</meta>
<meta property="rendition:spread">landscape</meta>
</metadata>
<metadata xmlns:opf="http://www.idpf.org/2007/opf" xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/">
<dc:title>${this.opts.title}</dc:title>
<dc:language>${this.opts.language || "en"}</dc:language>
<dc:creator>${this.opts.author}</dc:creator>
<dc:identifier id="bookid">${this.opts.id}</dc:identifier>
<dc:type>Image</dc:type>
<manifest>
<item id="fallback" href="fallback.xhtml" media-type="application/xhtml+xml" />
${hashes.map((t, i) => ` <item id="i${i}" href="${t}" fallback="fallback" media-type="image/${t.substr(t.lastIndexOf(".") + 1) === "jpg" ? "jpeg" : "png"}"/>`).join("\n")}
${hashes.map((t, i) => ` <item id="p${i}" href="${i}.xhtml" media-type="application/xhtml+xml" />`).join("\n")}
<meta property="dcterms:modified">${this.opts.updatedAt.toString().split("+")[0]}Z</meta>
<meta property="rendition:layout">pre-paginated</meta>
<meta property="rendition:orientation">portrait</meta>
<meta property="rendition:spread">landscape</meta>
</metadata>
<item id="ncxtoc" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
</manifest>
<manifest>
${this.hashes.map((t, i) => ` <item id="i${i}" href="${t}" fallback="fallback" media-type="image/${t.substr(t.lastIndexOf(".") + 1) === "jpg" ? "jpeg" : t.substr(t.lastIndexOf(".") + 1)}"/>`).join("\n")}
${this.hashes.map((t, i) => ` <item id="p${i}" href="${i}.xhtml" media-type="application/xhtml+xml" />`).join("\n")}
<spine toc="ncxtoc">
${hashes.map((t, i) => ` <itemref idref="p${i}" linear="yes" />`).join("\n")}
<itemref idref="fallback" linear="no" />
</spine>
<item id="ncxtoc" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
</manifest>
</package>`), true);
<spine toc="ncxtoc">
${this.hashes.map((t, i) => ` <itemref idref="p${i}" linear="yes" />`).join("\n")}
<itemref idref="fallback" linear="no" />
</spine>
const ncx = new ZipPassThrough("OEBPS/toc.ncx");
zip.add(ncx);
ncx.push(enc.encode(`<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns:ncx="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
<head>
<meta name="dtb:depth" content="1"/>
<meta name="dtb:totalPageCount" content="0"/>
<meta name="dtb:maxPageNumber" content="0"/>
</head>
<docTitle>
<text>${opts.title}</text>
</docTitle>
<docAuthor>
<text>Unknown</text>
</docAuthor>
</package>`), true);
}
<navMap>
${hashes.map((t, i) => `
<navPoint id="p${i}" playOrder="${i + 1}">
<navLabel>
<text>${opts.title} ${i}</text>
</navLabel>
<content src="${i}.xhtml"/>
</navPoint>
`).join("\n")}
</navMap>
</ncx>`), true);
toc() {
const ncx = new ZipPassThrough("OEBPS/toc.ncx");
this.zip.add(ncx);
ncx.push(enc.encode(`<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns:ncx="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
<head>
<meta name="dtb:depth" content="1"/>
<meta name="dtb:totalPageCount" content="0"/>
<meta name="dtb:maxPageNumber" content="0"/>
</head>
<docTitle>
<text>${this.opts.title}</text>
</docTitle>
<docAuthor>
<text>${this.opts.author}</text>
</docAuthor>
const fallback = new ZipPassThrough("OEBPS/fallback.xhtml");
zip.add(fallback);
fallback.push(enc.encode(`<?xml version="1.0" encoding="UTF-8"?>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head>
<title>${opts.title}</title>
</head>
<body>
<h2>This book cannot be opened on this device or using this program</h2>
<p>We're sorry</p>
<nav epub:type="toc">
<h1>Chapter list</h1>
<ol>
<li>
<a href="p0.xhtml">Chapter ${opts.chapter}</a>
</li>
</ol>
</nav>
</body>
</html>`), true);
return zip;
<navMap>
${this.opts.chapters.map((t, i) => t.links.map((link, i) => `
<navPoint id="p${i}" playOrder="${i + 1}">
<navLabel>
<text>${this.opts.title} Chapter ${t.number} Page ${i + 1}</text>
</navLabel>
<content src="${i}.xhtml"/>
</navPoint>
`)).flat().join("\n")}
</navMap>
</ncx>`), true);
}
}

24
src/util/report.js Normal file
View file

@ -0,0 +1,24 @@
export const base = "https://api.mangadex.network/";
/**
* reports method
* @param {object} body
* @param {string} body.url
* @param {boolean} body.success
* @param {number} body.bytes
* @param {number} body.duration
* @param {boolean} body.cached
*/
async function report(body) {
body.duration = parseInt(body.duration);
const resp = await fetch(base + "report", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: body ? JSON.stringify(body) : undefined
});
return await resp.json();
}
export default report;

View file

@ -1,4 +1,5 @@
const base = "https://cors-anywhere.danbulant.workers.dev/?https://api.mangadex.org/";
export const proxy = "https://cors-anywhere.danbulant.workers.dev/?";
export const base = proxy + "https://api.mangadex.org/";
function request(endpoint, query, type = "GET", body) {
return fetch(base + endpoint + (query ? "?" + query : ""), {