Skip to content

Commit

Permalink
preload
Browse files Browse the repository at this point in the history
  • Loading branch information
Oleg Koloskov committed Sep 3, 2024
1 parent 9bad415 commit e3422ab
Show file tree
Hide file tree
Showing 62 changed files with 299 additions and 50 deletions.
3 changes: 3 additions & 0 deletions Core/Configs/Options.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ public Options(IEnumerable<string> url) {

[Option("save-temp", Required = false, HelpText = "Сохранять временные файлы", Default = false)]
public bool SaveTemp { get; set; }

[Option("no-chapters", Required = false, HelpText = "Не загружать главы", Default = false)]
public bool NoChapters { get; set; }

public string ResourcesPath => "Patterns";
}
4 changes: 4 additions & 0 deletions Core/Logic/Getters/AcomicsGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ private static Author GetAuthor(HtmlDocument doc, Uri url) {
}

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, string bookId, string title, Uri url) {
if (Config.Options.NoChapters) {
return [];
}

var chapter = new Chapter {
Title = title
};
Expand Down
10 changes: 7 additions & 3 deletions Core/Logic/Getters/AuthorTodayGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,11 @@ protected override HttpRequestMessage GetImageRequestMessage(Uri uri) {
}

private async Task<IEnumerable<Chapter>> FillChapters(AuthorTodayBookDetails book) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var atChapter in await GetChapters(book)) {
var title = atChapter.Title.ReplaceNewLine();
Config.Logger.LogInformation($"Загружаю главу {title.CoverQuotes()}");
Expand All @@ -166,10 +170,10 @@ private async Task<IEnumerable<Chapter>> FillChapters(AuthorTodayBookDetails boo
chapter.Content = chapterDoc.DocumentNode.InnerHtml;
}

chapters.Add(chapter);
result.Add(chapter);
}

return chapters;
return result;
}

private async Task<IEnumerable<AuthorTodayChapter>> GetChapters(AuthorTodayBookDetails book) {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/BookInBookGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public override async Task<Book> Get(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri uri) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var bookChapter in GetToc(doc, uri)) {
var chapter = new Chapter();
Expand Down
4 changes: 4 additions & 0 deletions Core/Logic/Getters/BookRiverGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ private static Seria GetSeria(HtmlDocument doc, Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(Uri uri, string bookId) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

var internalId = await GetInternalBookId(bookId);

foreach (var bookChapter in await GetToc(internalId)) {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/BookinistGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ public override async Task<Book> Get(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var bookChapter in GetToc(doc, url)) {
Config.Logger.LogInformation($"Загружаю главу {bookChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/BookstabGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ private static string GetAnnotation(BookstabBook book) {

private async Task<IEnumerable<Chapter>> FillChapters(BookstabApiResponse response, Uri uri, string bookId) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var bookChapter in SliceToc(response.Book.ChaptersShow)) {
var chapter = new Chapter {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/BookstimeGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ private IEnumerable<UrlChapter> GetToc(HtmlDocument doc, Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in GetToc(doc, url)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/BookurukGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ private async Task<IEnumerable<UrlChapter>> GetToc(HtmlDocument doc, Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri uri) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in await GetToc(doc, uri)) {
var chapter = new Chapter {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/DarkNovelsGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ private async Task<Uri> GetMainUrl(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(string bookId) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var darkNovelsChapter in await GetToc(bookId)) {
Config.Logger.LogInformation($"Загружаю главу {darkNovelsChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/DesuGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ private static string GetTitle(HtmlDocument doc) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in GetToc(doc)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/DreameGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ private static T GetNextData<T>(HtmlDocument doc, string node) {

private async Task<IEnumerable<Chapter>> FillChapters(Uri url, string bookId) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in await GetToc(bookId)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/Fb2Top/Fb2TopGetterBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ public override async Task<Book> Get(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in GetToc(doc, url)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/FicbookGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ private Task<Image> GetCover(HtmlDocument doc, Uri bookUri) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url, string title) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var ficbookChapter in GetToc(doc, url, title)) {
Config.Logger.LogInformation($"Загружаю главу {ficbookChapter.Title.CoverQuotes()}");
Expand Down
12 changes: 8 additions & 4 deletions Core/Logic/Getters/FictionBookGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,11 @@ private async Task<int> GetPages(Uri url) {
}

private async Task<List<Chapter>> FillChapters(Uri url, string name) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

Chapter chapter = null;
var singleChapter = true;
var text = new StringBuilder();
Expand All @@ -118,14 +122,14 @@ private async Task<List<Chapter>> FillChapters(Uri url, string name) {
}
}
} else {
await AddChapter(chapters, chapter, text);
await AddChapter(result, chapter, text);
text.Clear();
chapter = CreateChapter(node.InnerText.HtmlDecode());
}
}
}

await AddChapter(chapters, chapter ?? CreateChapter(name), text);
return chapters;
await AddChapter(result, chapter ?? CreateChapter(name), text);
return result;
}
}
3 changes: 3 additions & 0 deletions Core/Logic/Getters/FreedlitGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ private async Task<IEnumerable<FreedlitChapter>> GetToc(FreedlitBook book) {

private async Task<IEnumerable<Chapter>> FillChapters(FreedlitBook book) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var freedlitChapter in await GetToc(book)) {
var chapter = new Chapter {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/Freedom/FreedomGetterBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ private IEnumerable<UrlChapter> GetToc(HtmlDocument doc, Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri uri) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in GetToc(doc, uri)) {
var chapter = new Chapter {
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/HogwartsNetGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ public override async Task<Book> Get(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in GetToc(doc, url)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
10 changes: 7 additions & 3 deletions Core/Logic/Getters/HotNovelPub/HotNovelPubGetterBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ private async Task<Uri> GetMainUrl(Uri url) {
}

private async Task<IEnumerable<Chapter>> FillChapters(ICollection<HotNovelPubChapter> toc) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var ezChapter in SliceToc(toc)) {
var title = ezChapter.Title.Trim();
Config.Logger.LogInformation($"Загружаю главу {title.CoverQuotes()}");
Expand All @@ -66,10 +70,10 @@ private async Task<IEnumerable<Chapter>> FillChapters(ICollection<HotNovelPubCha
chapter.Images = await GetImages(chapterDoc, SystemUrl);
chapter.Content = chapterDoc.DocumentNode.InnerHtml;

chapters.Add(chapter);
result.Add(chapter);
}

return chapters;
return result;
}

private async Task<HtmlDocument> GetChapter(HotNovelPubChapter ezChapter) {
Expand Down
4 changes: 4 additions & 0 deletions Core/Logic/Getters/HubBookGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ public override async Task<Book> Get(Uri url) {
}

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url, string title) {
if (Config.Options.NoChapters) {
return [];
}

var pages = int.Parse(doc.GetTextBySelector("span[itemprop=numberOfPages]"));
var text = new StringBuilder();

Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/IGramGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ private static Seria GetSeria(HtmlDocument doc, Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(Uri uri, HtmlDocument doc) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var bookChapter in await GetToc(doc)) {
Config.Logger.LogInformation($"Загружаю главу {bookChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/JaomixGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ public override async Task<Book> Get(Uri url) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc, Uri url) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var jaomixChapter in GetToc(doc, url)) {
Config.Logger.LogInformation($"Загружаю главу {jaomixChapter.Title.CoverQuotes()}");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,10 @@ protected override HttpRequestMessage GetImageRequestMessage(Uri uri) {
}

private async Task<IEnumerable<Chapter>> FillChapters(RanobeLibBookDetails book, string bid) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var socialChapter in await GetToc(book, bid)) {
var title = socialChapter.Name.ReplaceNewLine();
Expand All @@ -194,10 +197,10 @@ private async Task<IEnumerable<Chapter>> FillChapters(RanobeLibBookDetails book,
chapter.Images = await GetImages(chapterDoc, ImagesHost);
chapter.Content = chapterDoc.DocumentNode.InnerHtml;

chapters.Add(chapter);
result.Add(chapter);
}

return chapters;
return result;
}

private async Task<HtmlDocument> GetChapter(RanobeLibBookDetails book, SocialLibBookChapter chapter, string bid) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ private WindowData GetData(HtmlDocument doc) {

private async Task<IEnumerable<Chapter>> FillChapters(WindowData data, Uri url, string bidId) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

var branchId = string.IsNullOrWhiteSpace(bidId)
? data.Chapters.List
.GroupBy(c => c.BranchId)
Expand Down
12 changes: 8 additions & 4 deletions Core/Logic/Getters/LibboxGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ private async Task<int> GetPages(string bookId) {
}

private async Task<IEnumerable<Chapter>> FillChapters(string bookId, string title) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

Chapter chapter = null;
var singleChapter = true;
var text = new StringBuilder();
Expand All @@ -103,15 +107,15 @@ private async Task<IEnumerable<Chapter>> FillChapters(string bookId, string titl
}
}
} else {
await AddChapter(chapters, chapter, text);
await AddChapter(result, chapter, text);
text.Clear();
chapter = CreateChapter(node.InnerHtml.HtmlDecode());
}
}
}

await AddChapter(chapters, chapter ?? CreateChapter(title), text);
return chapters;
await AddChapter(result, chapter ?? CreateChapter(title), text);
return result;
}

private static Author GetAuthor(HtmlDocument doc, Uri url) {
Expand Down
12 changes: 8 additions & 4 deletions Core/Logic/Getters/LibkingGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,11 @@ private HtmlNode GetTitleNode(HtmlDocument doc) {
}

private async Task<List<Chapter>> FillChapters(HtmlDocument doc, Uri baseUrl, string name) {
var chapters = new List<Chapter>();
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

var fullText = new StringBuilder();
var i = 1;

Expand All @@ -118,12 +122,12 @@ private async Task<List<Chapter>> FillChapters(HtmlDocument doc, Uri baseUrl, st
} while (node.NextSibling != default && !IsTitle(node.NextSibling));

node = node.NextSibling;
await AddChapter(chapters, CreateChapter(title), chapterText);
await AddChapter(result, CreateChapter(title), chapterText);
}
} else {
await AddChapter(chapters, CreateChapter(name), fullText);
await AddChapter(result, CreateChapter(name), fullText);
}

return chapters;
return result;
}
}
3 changes: 3 additions & 0 deletions Core/Logic/Getters/LibstGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ private FormUrlEncodedContent GenerateAuthData(string token) {

private async Task<IEnumerable<Chapter>> FillChapters(HtmlDocument doc) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var bookChapter in GetToc(doc)) {
Config.Logger.LogInformation($"Загружаю главу {bookChapter.Title.CoverQuotes()}");
Expand Down
3 changes: 3 additions & 0 deletions Core/Logic/Getters/LitLifeGetter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ private async Task<IEnumerable<UrlChapter>> GetToc(string id) {

private async Task<IEnumerable<Chapter>> FillChapters(string id) {
var result = new List<Chapter>();
if (Config.Options.NoChapters) {
return result;
}

foreach (var urlChapter in await GetToc(id)) {
Config.Logger.LogInformation($"Загружаю главу {urlChapter.Title.CoverQuotes()}");
Expand Down
Loading

0 comments on commit e3422ab

Please sign in to comment.