Skip to content

Commit

Permalink
add all comic source to DB
Browse files Browse the repository at this point in the history
  • Loading branch information
@Haleydu
Haleydu committed Aug 30, 2020
1 parent 984ad2c commit 4214355
Show file tree
Hide file tree
Showing 44 changed files with 789 additions and 243 deletions.
7 changes: 4 additions & 3 deletions app/src/main/java/com/hiroshi/cimoc/core/Manga.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package com.hiroshi.cimoc.core;

import android.util.Log;

import com.hiroshi.cimoc.App;
import com.hiroshi.cimoc.manager.ChapterManager;
import com.hiroshi.cimoc.manager.SourceManager;
Expand All @@ -10,6 +8,8 @@
import com.hiroshi.cimoc.model.ImageUrl;
import com.hiroshi.cimoc.parser.Parser;
import com.hiroshi.cimoc.parser.SearchIterator;
import com.hiroshi.cimoc.rx.RxBus;
import com.hiroshi.cimoc.rx.RxEvent;

import org.json.JSONArray;
import org.json.JSONObject;
Expand Down Expand Up @@ -82,7 +82,8 @@ public void call(Subscriber<? super List<Chapter>> subscriber) {
comic.setUrl(parser.getUrl(comic.getCid()));
Request request = parser.getInfoRequest(comic.getCid());
String html = getResponseBody(App.getHttpClient(), request);
parser.parseInfo(html, comic);
Comic newComic = parser.parseInfo(html, comic);
RxBus.getInstance().post(new RxEvent(RxEvent.EVENT_COMIC_UPDATE_INFO, newComic));
request = parser.getChapterRequest(html, comic.getCid());
if (request != null) {
html = getResponseBody(App.getHttpClient(), request);
Expand Down
10 changes: 0 additions & 10 deletions app/src/main/java/com/hiroshi/cimoc/parser/MangaParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ protected void initUrlFilterList() {
// filter.add(new UrlFilter("manhua.dmzj.com", "/(\\w+)", 1));
}

@Override
public List<Chapter> parseChapter(String html, Comic comic) throws JSONException{
return null;
}

@Override
public List<Chapter> parseChapter(String html) throws JSONException {
return null;
Expand All @@ -53,11 +48,6 @@ public List<ImageUrl> parseImages(String html) throws Manga.NetworkErrorExceptio
return null;
}

@Override
public List<ImageUrl> parseImages(String html, Chapter chapter) throws Manga.NetworkErrorException, JSONException {
return null;
}

@Override
public Request getChapterRequest(String html, String cid) {
return null;
Expand Down
3 changes: 2 additions & 1 deletion app/src/main/java/com/hiroshi/cimoc/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ public interface Parser {
* @param html 页面源代码
* @param comic 漫画实体类，需要设置其中的字段
*/
void parseInfo(String html, Comic comic) throws UnsupportedEncodingException;
Comic parseInfo(String html, Comic comic) throws UnsupportedEncodingException;


/**
* 章节列表的 HTTP 请求，若在 {@link #parseInfo} 中可以解析出章节列表，返回 null，代表不用再次解析
Expand Down
26 changes: 20 additions & 6 deletions app/src/main/java/com/hiroshi/cimoc/source/Animx2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public Request getInfoRequest(String cid) {
}

@Override
public void parseInfo(String html, Comic comic) throws UnsupportedEncodingException {
public Comic parseInfo(String html, Comic comic) throws UnsupportedEncodingException {
Node body = new Node(html);
String title = body.text("div.position > strong");
String cover = "http://www.2animx.com/" + body.src("dl.mh-detail > dt > a > img");
Expand All @@ -88,18 +88,30 @@ public void parseInfo(String html, Comic comic) throws UnsupportedEncodingExcept
String intro = body.text(".mh-introduce");
boolean status = false;
comic.setInfo(title, cover, update, intro, author, status);
return comic;
}

@Override
public List<Chapter> parseChapter(String html) {
public List<Chapter> parseChapter(String html, Comic comic) {
List<Chapter> list = new LinkedList<>();
int i=0;
for (Node node : new Node(html).list("div#oneCon2 > ul > li")) {
Long sourceComic=null;
if (comic.getId() == null) {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + "00");
} else {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + comic.getId());
}
Long id = Long.parseLong(sourceComic+"000"+i);

String title = node.attr("a", "title");
Matcher mTitle = Pattern.compile("\\d+").matcher(title);
title = mTitle.find() ? mTitle.group() : title;
String path2 = node.href("a");
String path = node.hrefWithSplit("a", 0);
list.add(new Chapter(title, path));

//list.add(new Chapter(title, path));
list.add(new Chapter(id, sourceComic, title, path));
i++;
}
return list;
}
Expand All @@ -117,13 +129,15 @@ public Request getImagesRequest(String cid, String path) {
}

@Override
public List<ImageUrl> parseImages(String html) {
public List<ImageUrl> parseImages(String html, Chapter chapter) {
List<ImageUrl> list = new ArrayList<>();
Matcher pageMatcher = Pattern.compile("id=\"total\" value=\"(.*?)\"").matcher(html);
if (!pageMatcher.find()) return null;
int page = Integer.parseInt(pageMatcher.group(1));
for (int i = 1; i <= page; ++i) {
list.add(new ImageUrl(i, StringUtils.format("%s-p-%d", _path, i), true));
Long comicChapter = chapter.getId();
Long id = Long.parseLong(comicChapter + "000" + i);
list.add(new ImageUrl(id, comicChapter, i, StringUtils.format("%s-p-%d", _path, i), true));
}
return list;
}
Expand Down
25 changes: 20 additions & 5 deletions app/src/main/java/com/hiroshi/cimoc/source/BaiNian.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public Request getInfoRequest(String cid) {
}

@Override
public void parseInfo(String html, Comic comic) {
public Comic parseInfo(String html, Comic comic) {
Node body = new Node(html);
String cover = body.attr("div.dbox > div.img > mip-img", "src");
String title = body.text("div.dbox > div.data > h4");
Expand All @@ -98,15 +98,28 @@ public void parseInfo(String html, Comic comic) {
String update = body.text("div.dbox > div.data > p.act").substring(3, 13).trim();
boolean status = isFinish(body.text("span.list_item"));
comic.setInfo(title, cover, update, intro, author, status);
return comic;
}

@Override
public List<Chapter> parseChapter(String html) {
public List<Chapter> parseChapter(String html, Comic comic) {
List<Chapter> list = new LinkedList<>();
int i=0;
for (Node node : new Node(html).list("div.tabs_block > ul > li > a")) {
Long sourceComic=null;
if (comic.getId() == null) {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + "00");
} else {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + comic.getId());
}
Long id = Long.parseLong(sourceComic+"000"+i);

String title = node.text();
String path = node.href();
list.add(new Chapter(title, path));

//list.add(new Chapter(title, path));
list.add(new Chapter(id, sourceComic, title, path));
i++;
}
return list;
}
Expand All @@ -118,7 +131,7 @@ public Request getImagesRequest(String cid, String path) {
}

@Override
public List<ImageUrl> parseImages(String html) {
public List<ImageUrl> parseImages(String html,Chapter chapter) {
List<ImageUrl> list = new LinkedList<>();
String host = StringUtils.match("src=\"(.*?)\\/upload", html, 1);
String path_str = StringUtils.match("z_img=\'\\[(.*?)\\]\'", html, 1);
Expand All @@ -127,7 +140,9 @@ public List<ImageUrl> parseImages(String html) {
String[] array = path_str.split(",");
for (int i = 0; i != array.length; ++i) {
String path = array[i].replace("\"","").replace("\\","");
list.add(new ImageUrl(i + 1, host+"/"+path, false));
Long comicChapter = chapter.getId();
Long id = Long.parseLong(comicChapter + "000" + i);
list.add(new ImageUrl(id,comicChapter,i + 1, host+"/"+path, false));
}
} catch (Exception e) {
e.printStackTrace();
Expand Down
24 changes: 19 additions & 5 deletions app/src/main/java/com/hiroshi/cimoc/source/BuKa.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public Request getInfoRequest(String cid) {
}

@Override
public void parseInfo(String html, Comic comic) throws UnsupportedEncodingException {
public Comic parseInfo(String html, Comic comic) throws UnsupportedEncodingException {
Node body = new Node(html);
String title = body.text("p.mangadir-glass-name");
String cover = body.src(".mangadir-glass-img > img");
Expand All @@ -109,6 +109,7 @@ public void parseInfo(String html, Comic comic) throws UnsupportedEncodingExcept
String intro = body.text("span.description_intro");
boolean status = isFinish("连载中");//todo: fix here
comic.setInfo(title, cover, update, intro, author, status);
return comic;
}

// @Override
Expand All @@ -120,12 +121,23 @@ public void parseInfo(String html, Comic comic) throws UnsupportedEncodingExcept
// }

@Override
public List<Chapter> parseChapter(String html) {
public List<Chapter> parseChapter(String html, Comic comic) {
List<Chapter> list = new LinkedList<>();
int i=0;
for (Node node : new Node(html).list("div.chapter-center > a")) {
Long sourceComic=null;
if (comic.getId() == null) {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + "00");
} else {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + comic.getId());
}
Long id = Long.parseLong(sourceComic+"000"+i);

String title = node.text();
String path = node.href().split("/")[3];
list.add(new Chapter(title, path));

list.add(new Chapter(id, sourceComic, title, path));
i++;
}
return list;
}
Expand All @@ -140,14 +152,16 @@ public Request getImagesRequest(String cid, String path) {
}

@Override
public List<ImageUrl> parseImages(String html) {
public List<ImageUrl> parseImages(String html,Chapter chapter) {
List<ImageUrl> list = new LinkedList<>();
Matcher m = Pattern.compile("<img class=\"lazy\" data-original=\"(http.*?jpg)\" />").matcher(html);
if (m.find()) {
try {
int i = 0;
do {
list.add(new ImageUrl(++i, StringUtils.match("http.*jpg", m.group(0), 0), false));
Long comicChapter = chapter.getId();
Long id = Long.parseLong(comicChapter + "000" + i);
list.add(new ImageUrl(id,comicChapter,++i, StringUtils.match("http.*jpg", m.group(0), 0), false));
} while (m.find());
} catch (Exception e) {
e.printStackTrace();
Expand Down
24 changes: 19 additions & 5 deletions app/src/main/java/com/hiroshi/cimoc/source/CCMH.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public Request getInfoRequest(String cid) {
}

@Override
public void parseInfo(String html, Comic comic) {
public Comic parseInfo(String html, Comic comic) {
Node body = new Node(html);
String intro = body.text(".intro");
String title = body.text(".other > div > strong");
Expand All @@ -111,16 +111,28 @@ public void parseInfo(String html, Comic comic) {
.replace("[", "").replace("]", "");
boolean status = isFinish(body.textWithSplit(".other", "\\s+|：", 10));
comic.setInfo(title, cover, update, intro, author, status);
return comic;
}

@Override
public List<Chapter> parseChapter(String html) {
public List<Chapter> parseChapter(String html, Comic comic) {
List<Chapter> list = new LinkedList<>();
Node body = new Node(html);
int i=0;
for (Node node : body.list(".list > a")) {
Long sourceComic=null;
if (comic.getId() == null) {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + "00");
} else {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + comic.getId());
}
Long id = Long.parseLong(sourceComic+"000"+i);

String title = node.attr("title");
String path = node.hrefWithSplit(2);
list.add(new Chapter(title, path));
//list.add(new Chapter(title, path));
list.add(new Chapter(id, sourceComic, title, path));
i++;
}

return Lists.reverse(list);
Expand All @@ -140,7 +152,7 @@ public Request getImagesRequest(String cid, String path) {
}

@Override
public List<ImageUrl> parseImages(String html) {
public List<ImageUrl> parseImages(String html, Chapter chapter) {
List<ImageUrl> list = new LinkedList<>();

//find image count
Expand All @@ -152,7 +164,9 @@ public List<ImageUrl> parseImages(String html) {
}

for (int i = 0; i < pageCount; i++) {
list.add(new ImageUrl(i, StringUtils.format("http://m.ccmh6.com/manhua/%s/%s.html?p=%d", _cid, _path, i + 1), true));
Long comicChapter = chapter.getId();
Long id = Long.parseLong(comicChapter + "000" + i);
list.add(new ImageUrl(id, comicChapter, i, StringUtils.format("http://m.ccmh6.com/manhua/%s/%s.html?p=%d", _cid, _path, i + 1), true));
}
return list;
}
Expand Down
24 changes: 19 additions & 5 deletions app/src/main/java/com/hiroshi/cimoc/source/CCTuku.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public Request getInfoRequest(String cid) {
}

@Override
public void parseInfo(String html, Comic comic) {
public Comic parseInfo(String html, Comic comic) {
Node body = new Node(html);
String title = body.text("div.detailTop > div.content > div.info > p.title");
String cover = body.src("div.detailTop > div.content > img");
Expand All @@ -84,17 +84,29 @@ public void parseInfo(String html, Comic comic) {
String intro = body.text("div.detailContent > p:eq(1)");
// FIXME 手机版页面好像获取不到状态 电脑板页面太大不想用 暂时先固定为连载吧
comic.setInfo(title, cover, update, intro, author, false);
return comic;
}

@Override
public List<Chapter> parseChapter(String html) {
public List<Chapter> parseChapter(String html, Comic comic) {
List<Chapter> list = new LinkedList<>();
Node body = new Node(html);
int i=0;
for (Node node : body.list("#chapter > div > div > ul > li > a")) {
Long sourceComic=null;
if (comic.getId() == null) {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + "00");
} else {
sourceComic = Long.parseLong(comic.getSource() + sourceToComic + comic.getId());
}
Long id = Long.parseLong(sourceComic+"000"+i);

String title = node.text();
// String path = StringUtils.split(node.href(), "/", 3);
String path = node.hrefWithSplit(2);
list.add(new Chapter(title, path));
//list.add(new Chapter(title, path));
list.add(new Chapter(id, sourceComic, title, path));
i++;
}
return list;
}
Expand All @@ -113,14 +125,16 @@ public Request getImagesRequest(String cid, String path) {
}

@Override
public List<ImageUrl> parseImages(String html) {
public List<ImageUrl> parseImages(String html, Chapter chapter) {
// TODO 好像拿不到总页数 GG
List<ImageUrl> list = new ArrayList<>();
// Node body = new Node(html);
// int page = Integer.parseInt(body.attr("#hdPageCount", "value"));//max pages unknow...
int page = 10;
for (int i = 1; i <= page; ++i) {
list.add(new ImageUrl(i, StringUtils.format("http://m.tuku.cc/comic/%s/%s/p%s/", _cid, _path, i), true));
Long comicChapter = chapter.getId();
Long id = Long.parseLong(comicChapter + "000" + i);
list.add(new ImageUrl(id, comicChapter, i, StringUtils.format("http://m.tuku.cc/comic/%s/%s/p%s/", _cid, _path, i), true));
}
return list;
}
Expand Down
Loading

0 comments on commit 4214355

Please sign in to comment.