Skip to content

Commit

Permalink
mangaAlbums合并到albums
Browse files Browse the repository at this point in the history
  • Loading branch information
@PhiFever
PhiFever committed Nov 29, 2024
1 parent db7827b commit 0c34ff3
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 135 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ AfdianToMarkdown.exe -l "文件路径"
```

### 更新日志
### v0.4.0
增加了对于漫画作品集的支持（暂未发布）

#### v0.3.0
1. 修改默认域名为`afdian.com`
Expand Down
119 changes: 55 additions & 64 deletions afdian/afdian.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,40 @@ const (
)

type Album struct {
AlbumName string `json:"albumName"`
AlbumUrl string `json:"albumUrl"`
AlbumName string
AlbumUrl string
}

type AlbumPost interface {
GetName() string
GetUrl() string
}

type Article struct {
ArticleName string `json:"articleName"`
ArticleUrl string `json:"articleUrl"`
Name string
Url string
}

func (a Article) GetName() string {
return a.Name
}

func (a Article) GetUrl() string {
return a.Url
}

type Manga struct {
MangaName string `json:"albumName"`
MangaUrl string `json:"mangaUrl"`
Pictures []string `json:"pics"`
Name string
Url string
Pictures []string
}

func (m Manga) GetName() string {
return m.Name
}

func (m Manga) GetUrl() string {
return m.Url
}

// Cookie 从 Chrome 中使用cookie master导出的 Cookies
Expand Down Expand Up @@ -116,8 +137,6 @@ func buildAfdianHeaders(cookieString string, referer string) http.Header {
"authority": {"afdian.com"},
"accept": {"accept", "application/json, text/plain, */*"},
"accept-language": {"zh-CN,zh;q=0.9,en;q=0.8"},
"afd-fe-version": {"20220508"},
"afd-stat-id": {"c78521949a7c11ee8c2452540025c377"},
"cache-control": {"no-cache"},
"cookie": {cookieString},
"dnt": {"1"},
Expand Down Expand Up @@ -175,7 +194,7 @@ func GetAuthorMotionUrlList(userName string, cookieString string, prevPublishSn
articleId := value.Get("post_id").String()
articleUrl, _ := url.JoinPath(Host, "post", articleId)
articleName := value.Get("title").String()
authorArticleList = append(authorArticleList, Article{ArticleName: utils.ToSafeFilename(articleName), ArticleUrl: articleUrl})
authorArticleList = append(authorArticleList, Article{Name: utils.ToSafeFilename(articleName), Url: articleUrl})
return true
})

Expand Down Expand Up @@ -203,75 +222,47 @@ func GetAlbumList(userId string, referer string, cookieString string) (albumList
return albumList
}

// GetAlbumArticleList 获取作品集的所有文章
func GetAlbumArticleList(albumId string, authToken string) (albumArticleList []Article) {
//log.Println("albumId:", albumId)
func GetAlbumPostList(albumId string, authToken string) (albumPostList []AlbumPost) {
postCountApiUrl := fmt.Sprintf("%s/api/user/get-album-info?album_id=%s", Host, albumId)
authTokenCookie := fmt.Sprintf("auth_token=%s", authToken)
referer := fmt.Sprintf("%s/album/%s", Host, albumId)

postCountBodyText := NewRequestGet(postCountApiUrl, authTokenCookie, referer)
postCount := gjson.GetBytes(postCountBodyText, "data.album.post_count").Int()
//log.Println("postCount:", postCount)

var i int64
for i = 0; i < postCount; i += 10 {
apiUrl := fmt.Sprintf("%s/api/user/get-album-post?album_id=%s&lastRank=%d&rankOrder=asc&rankField=rank", Host, albumId, i)
body := NewRequestGet(apiUrl, authTokenCookie, referer)

albumArticleListJson := gjson.GetBytes(body, "data.list")
albumArticleListJson.ForEach(func(key, value gjson.Result) bool {
//fmt.Println(value.Get("title").String())
//fmt.Println(value.Get("post_id").String())
albumPostListJson := gjson.GetBytes(body, "data.list")
albumPostListJson.ForEach(func(key, value gjson.Result) bool {
postId := value.Get("post_id").String()
postUrl, _ := url.JoinPath(Host, "album", albumId, postId)
albumArticleList = append(albumArticleList, Article{ArticleName: utils.ToSafeFilename(value.Get("title").String()), ArticleUrl: postUrl})
return true
})
}

return albumArticleList
}

// TODO：和上一个函数合并
func GetAlbumMangaList(albumId string, authToken string) (mangaList []Manga) {
//log.Println("albumId:", albumId)
postCountApiUrl := fmt.Sprintf("%s/api/user/get-album-info?album_id=%s", Host, albumId)
authTokenCookie := fmt.Sprintf("auth_token=%s", authToken)
referer := fmt.Sprintf("%s/album/%s", Host, albumId)

postCountBodyText := NewRequestGet(postCountApiUrl, authTokenCookie, referer)
postCount := gjson.GetBytes(postCountBodyText, "data.album.post_count").Int()
//log.Println("postCount:", postCount)

var i int64
for i = 0; i < postCount; i += 10 {
apiUrl := fmt.Sprintf("%s/api/user/get-album-post?album_id=%s&lastRank=%d&rankOrder=asc&rankField=rank", Host, albumId, i)
body := NewRequestGet(apiUrl, authTokenCookie, referer)

albumArticleListJson := gjson.GetBytes(body, "data.list")
albumArticleListJson.ForEach(func(key, value gjson.Result) bool {
//fmt.Println(value.Get("title").String())
//fmt.Println(value.Get("post_id").String())
postId := value.Get("post_id").String()
postUrl, _ := url.JoinPath(Host, "album", albumId, postId)
mangaList = append(mangaList,
Manga{
MangaName: utils.ToSafeFilename(value.Get("title").String()),
MangaUrl: postUrl,
Pictures: func() []string {
var pictures []string
for _, result := range value.Get("pics").Array() {
pictures = append(pictures, result.String()) // 提取每个结果的字符串值
}
return pictures
}(),
if value.Get("pics").Exists() {
// 如果有图片，则创建一个 Manga 对象
var pictures []string
for _, result := range value.Get("pics").Array() {
pictures = append(pictures, result.String())
}
albumPostList = append(albumPostList, Manga{
Name: utils.ToSafeFilename(value.Get("title").String()),
Url: postUrl,
Pictures: pictures,
})
} else {
// 否则创建一个 Article 对象
albumPostList = append(albumPostList, Article{
Name: utils.ToSafeFilename(value.Get("title").String()),
Url: postUrl,
})
}
return true
})
}

return mangaList
return albumPostList
}

// GetArticleContent 获取文章正文内容
Expand Down Expand Up @@ -375,7 +366,7 @@ func SaveMangaIfNotExist(filePath string, manga Manga, authToken string, convert
if err := os.MkdirAll(assetsDir, os.ModePerm); err != nil {
return fmt.Errorf("create assets directory error: %v", err)
}
content := GetArticleContent(manga.MangaUrl, authToken, converter)
content := GetArticleContent(manga.Url, authToken, converter)
picContent := ""
// 下载并保存图片到本地
for i, pictureUrl := range manga.Pictures {
Expand All @@ -384,10 +375,10 @@ func SaveMangaIfNotExist(filePath string, manga Manga, authToken string, convert
if ext == "" {
ext = ".jpg" // 默认扩展名
}
localFileName := fmt.Sprintf("%s_%d%s", utils.ToSafeFilename(manga.MangaName), i, ext)
localFileName := fmt.Sprintf("%s_%d%s", utils.ToSafeFilename(manga.Name), i, ext)
localFilePath := filepath.Join(assetsDir, localFileName)

log.Printf("Downloading picture in manga %s: %s", manga.MangaName, pictureUrl)
log.Printf("Downloading picture in manga %s: %s", manga.Name, pictureUrl)
// 使用requests下载图片
err := requests.
URL(pictureUrl).
Expand All @@ -407,9 +398,9 @@ func SaveMangaIfNotExist(filePath string, manga Manga, authToken string, convert
picContent += fmt.Sprintf("![image](%s)\n", relPath)
}

commentString, hotCommentString := GetArticleComment(manga.MangaUrl, authToken)
commentString, hotCommentString := GetArticleComment(manga.Url, authToken)
//Refer中需要把articleUrl中的post替换成p才能在浏览器正常访问
articleContent := "## " + manga.MangaName + "\n\n### Refer\n\n" + strings.Replace(manga.MangaUrl, "post", "p", 1) + "\n\n### 正文\n\n" + fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s", content, picContent, hotCommentString, commentString)
articleContent := "## " + manga.Name + "\n\n### Refer\n\n" + strings.Replace(manga.Url, "post", "p", 1) + "\n\n### 正文\n\n" + fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s", content, picContent, hotCommentString, commentString)
//log.Println("articleContent:", articleContent)
if err := os.WriteFile(filePath, []byte(articleContent), os.ModePerm); err != nil {
return err
Expand Down
20 changes: 10 additions & 10 deletions afdian/afdian_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"testing"
)

const q9adg_id = "3f49234e3e8f11eb8f6152540025c377"
const q9adgId = "3f49234e3e8f11eb8f6152540025c377"

var cookieString, authToken string

Expand Down Expand Up @@ -45,7 +45,7 @@ func TestGetAuthorId(t *testing.T) {
referer: Host,
cookieString: cookieString,
},
want: q9adg_id,
want: q9adgId,
},
{name: "深海巨狗",
args: args{
Expand All @@ -70,18 +70,18 @@ func TestGetAuthorMotionUrlList(t *testing.T) {
prevPublishSn string
}
tests := []struct {
name string
args args
wantArticleList []Article
wantNextPublish_sn string
name string
args args
wantArticleList []Article
wantNextPublishSn string
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
authorArticleList, nextPublish_sn := GetAuthorMotionUrlList(tt.args.userName, tt.args.cookieString, tt.args.prevPublishSn)
authorArticleList, nextPublishSn := GetAuthorMotionUrlList(tt.args.userName, tt.args.cookieString, tt.args.prevPublishSn)
assert.Equalf(t, tt.wantArticleList, authorArticleList, "GetAuthorMotionUrlList(%v, %v, %v)", tt.args.userName, tt.args.cookieString, tt.args.prevPublishSn)
assert.Equalf(t, tt.wantNextPublish_sn, nextPublish_sn, "GetAuthorMotionUrlList(%v, %v, %v)", tt.args.userName, tt.args.cookieString, tt.args.prevPublishSn)
assert.Equalf(t, tt.wantNextPublishSn, nextPublishSn, "GetAuthorMotionUrlList(%v, %v, %v)", tt.args.userName, tt.args.cookieString, tt.args.prevPublishSn)
})
}
}
Expand All @@ -100,7 +100,7 @@ func TestGetAlbumList(t *testing.T) {
{
name: "q9adg",
args: args{
userId: q9adg_id,
userId: q9adgId,
referer: Host,
cookieString: cookieString,
},
Expand Down Expand Up @@ -143,7 +143,7 @@ func TestGetAlbumArticleList(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, GetAlbumArticleList(tt.args.albumId, tt.args.authToken), "GetAlbumArticleList(%v, %v)", tt.args.albumId, tt.args.authToken)
assert.Equalf(t, tt.want, GetAlbumPostList(tt.args.albumId, tt.args.authToken), "GetAlbumArticleList(%v, %v)", tt.args.albumId, tt.args.authToken)
})
}
}
Expand Down
70 changes: 18 additions & 52 deletions afdian/album/album.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,76 +15,42 @@ import (

func GetAlbums(authorName string, cookieString string, authToken string) error {
albumHost, _ := url.JoinPath(afdian.Host, "a", authorName, "album")
// 获取作者的所有作品集
log.Println("albumHost:", albumHost)

userId := afdian.GetAuthorId(authorName, albumHost, cookieString)
//log.Println("userId:", userId)
albumList := afdian.GetAlbumList(userId, albumHost, cookieString)
//log.Println("albumList:", utils.ToJSON(albumList))

converter := md.NewConverter("", true, nil)
for _, album := range albumList {
log.Println("Find album: ", album.AlbumName)
//获取作品集的所有文章
//album.AlbumUrl会类似于 https://afdian.com/album/xyz
re := regexp.MustCompile("^.*/album/")
albumId := re.ReplaceAllString(album.AlbumUrl, "")
albumArticleList := afdian.GetAlbumArticleList(albumId, authToken)
albumPostList := afdian.GetAlbumPostList(albumId, authToken)
time.Sleep(time.Millisecond * time.Duration(afdian.DelayMs))

//log.Println("albumArticleList:", utils.ToJSON(albumArticleList))
//log.Println(len(albumArticleList))

_ = os.MkdirAll(path.Join(authorName, album.AlbumName), os.ModePerm)

for i, article := range albumArticleList {
filePath := path.Join(utils.GetExecutionPath(), authorName, album.AlbumName, cast.ToString(i)+"_"+article.ArticleName+".md")
for i, post := range albumPostList {
filePath := path.Join(utils.GetExecutionPath(), authorName, album.AlbumName, cast.ToString(i)+"_"+post.GetName()+".md")
log.Println("Saving file:", filePath)

if err := afdian.SaveContentIfNotExist(article.ArticleName, filePath, article.ArticleUrl, authToken, converter); err != nil {
return err
if manga, ok := post.(afdian.Manga); ok {
// 如果转换成功，说明当前元素是 Manga 类型
//fmt.Println("Manga Name:", manga.Name)
//fmt.Println("Manga URL:", manga.Url)
//fmt.Println("Manga Pictures:", manga.Pictures)
if err := afdian.SaveMangaIfNotExist(filePath, manga, authToken, converter); err != nil {
return err
}
} else if article, ok := post.(afdian.Article); ok {
if err := afdian.SaveContentIfNotExist(article.Name, filePath, article.Url, authToken, converter); err != nil {
return err
}
} else {
log.Fatal("Unknown post type")
}
//break
}
break

}
return nil
}

func GetMangaAlbums(authorName string, cookieString string, authToken string) error {
albumHost, _ := url.JoinPath(afdian.Host, "a", authorName, "album")
// 获取作者的所有作品集
log.Println("albumHost:", albumHost)

userId := afdian.GetAuthorId(authorName, albumHost, cookieString)
//log.Println("userId:", userId)
albumList := afdian.GetAlbumList(userId, albumHost, cookieString)
//log.Println("albumList:", utils.ToJSON(albumList))

converter := md.NewConverter("", true, nil)
for _, album := range albumList {
log.Println("Find album: ", album.AlbumName)
//获取作品集的所有文章
//album.AlbumUrl会类似于 https://afdian.com/album/xyz
re := regexp.MustCompile("^.*/album/")
albumId := re.ReplaceAllString(album.AlbumUrl, "")
albumMangaList := afdian.GetAlbumMangaList(albumId, authToken)
//log.Println("albumMangaList:", albumMangaList)

time.Sleep(time.Millisecond * time.Duration(afdian.DelayMs))

_ = os.MkdirAll(path.Join(authorName, album.AlbumName), os.ModePerm)

for _, manga := range albumMangaList {
filePath := path.Join(utils.GetExecutionPath(), authorName, album.AlbumName, manga.MangaName+".md")
log.Println("Saving file:", filePath)

if err := afdian.SaveMangaIfNotExist(filePath, manga, authToken, converter); err != nil {
return err
}
}
}

return nil
}
4 changes: 2 additions & 2 deletions afdian/motion/motion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ func GetMotions(authorName string, cookieString string, authToken string) error

converter := md.NewConverter("", true, nil)
for i, article := range articleList {
filePath := path.Join(utils.GetExecutionPath(), authorName, authorDir, cast.ToString(i)+"_"+article.ArticleName+".md")
filePath := path.Join(utils.GetExecutionPath(), authorName, authorDir, cast.ToString(i)+"_"+article.Name+".md")
log.Println("Saving file:", filePath)
if err := afdian.SaveContentIfNotExist(article.ArticleName, filePath, article.ArticleUrl, authToken, converter); err != nil {
if err := afdian.SaveContentIfNotExist(article.Name, filePath, article.Url, authToken, converter); err != nil {
return err
}
//break
Expand Down
7 changes: 0 additions & 7 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,6 @@ func main() {
return album.GetAlbums(authorName, cookieString, authToken)
},
},
{
Name: "mangaAlbums",
Usage: "下载指定作者的所有漫画作品集",
Action: func(c *cli.Context) error {
return album.GetMangaAlbums(authorName, cookieString, authToken)
},
},
{
Name: "update",
Usage: "更新所有已经下载的作者的动态和作品集",
Expand Down

0 comments on commit 0c34ff3

Please sign in to comment.