Merge pull request #57 from mnhkahn/refactor_dao

Refactor dao

.env

@@ -0,0 +1,2 @@
+BINARY=maodou
+TEST_PACKAGES=github.com/mnhkahn/maodou/dao

Makefile

@@ -0,0 +1,39 @@
+include .env
+
+d := $(shell date)
+branch := $(shell git branch | grep \* | cut -d ' ' -f2)
+
+.PHONY: help
+all: help
+help: Makefile
+	@echo
+	@echo " Choose a command run in ${BINARY}:"
+	@echo
+	@sed -n 's/^##//p' $< | column -t -s ':' |  sed -e 's/^/ /'
+	@echo
+
+## pull: Revert & pull code from remote by current branch.
+pull:
+	git reset --hard
+	git pull origin ${branch}
+
+## push: Auto commit & push code to remote by current branch.
+push: version
+	git commit -am "chore: date is $d."
+	git push origin ${branch}
+
+## cover: Run all test code & print cover profile.
+cover:
+	go test -coverprofile=/tmp/${BINARY}-makecover.out ${TEST_PACKAGES} && go tool cover -html=/tmp/${BINARY}-makecover.out
+
+## version: Show current code version.
+version:
+	@git remote -v
+	@echo branch: $(branch)
+
+## chglog: Run git-chglog & save to CHANGELOG
+chglog:
+	@git-chglog -o CHANGELOG.md
+	git commit -am "add chglog: date is $d."
+	git push origin ${branch}
+

dao/dao.go

@@ -8,18 +8,12 @@ import (
 
 type DaoContainer interface {
 	AddResult(p *Result) error
-	AddResults(p []Result)
 	DelResult(id interface{})
-	DelResults(source string)
-	UpdateResult(p *Result)
-	AddOrUpdate(p *Result)
 	GetResults() ([]*Result, error)
 	GetResultById(id uint64) (*Result, error)
-	GetResultByLink(url string) *Result
-	GetResult(author, sort string, limit, start int) []Result
-	IsResultUpdate(p *Result) bool
 	Search(q string, limit, start int) (int, float64, []Result)
 	Debug(is_debug bool)
+	Close() error
 }
 
 type Dao interface {

dao/peanut_dao.go

@@ -72,26 +72,10 @@ func (this *PeanutContainer) toResults(ps []*index.Document) []Result {
 	return res
 }
 
-func (this *PeanutContainer) AddResults(p []Result) {
-
-}
-
 func (this *PeanutContainer) DelResult(id interface{}) {
 
 }
 
-func (this *PeanutContainer) DelResults(source string) {
-
-}
-
-func (this *PeanutContainer) UpdateResult(p *Result) {
-
-}
-
-func (this *PeanutContainer) AddOrUpdate(p *Result) {
-	this.AddResult(p)
-}
-
 func (this *PeanutContainer) GetResults() ([]*Result, error) {
 	return nil, nil
 }
@@ -100,32 +84,24 @@ func (this *PeanutContainer) GetResultById(id uint64) (*Result, error) {
 	return nil, nil
 }
 
-func (this *PeanutContainer) GetResultByLink(url string) *Result {
-	p := new(Result)
-	return p
-}
-
-func (this *PeanutContainer) GetResult(author, sort string, limit, start int) []Result {
-	return nil
-}
-
-func (this *PeanutContainer) IsResultUpdate(p *Result) bool {
-	return false
-}
-
 func (this *PeanutContainer) Search(q string, limit, start int) (int, float64, []Result) {
 	cnt, res, err := this.ind.Search(&index.Param{
 		Query:  q,
 		Offset: start,
 		Size:   limit,
 	})
+
 	if err != nil {
 		logger.Warn(err)
 	}
 
 	return cnt, 0, this.toResults(res)
 }
 
+func (this *PeanutContainer) Close() error {
+	return this.ind.Close()
+}
+
 func init() {
 	Register("peanut", &PeanutDao{})
 }

demo/cyeamblog.go

@@ -7,8 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mnhkahn/gogogo/logger"
-
 	"github.com/PuerkitoBio/goquery"
 	"github.com/mnhkahn/maodou"
 	"github.com/mnhkahn/maodou/models"
@@ -18,31 +16,18 @@ type CyeamBlogCrawler struct {
 	maodou.MaoDou
 }
 
-func (this *CyeamBlogCrawler) Start() {
-	resp, err := this.Cawl("http://blog.cyeam.com/")
-	if err != nil {
-		logger.Warn(err)
-	} else {
-		this.Index(resp)
-	}
-}
-
-func (this *CyeamBlogCrawler) Index(resp *maodou.Response) {
+func (this *CyeamBlogCrawler) Index(resp *maodou.Response, jobs chan string) error {
 	resp.Doc(`#content > div > div > div > h2 > a`).Each(func(i int, s *goquery.Selection) {
 		href, has := s.Attr("href")
-		//logger.Infof("AA %+v", href)
 		if has {
-			resp, err := this.Cawl("http://blog.cyeam.com/" + href)
-			if err != nil {
-				logger.Warn(err)
-			} else {
-				this.Detail(resp)
-			}
+			this.AddJob("http://blog.cyeam.com/" + href)
 		}
 	})
+	return nil
 }
 
-func (this *CyeamBlogCrawler) Detail(resp *maodou.Response) {
+func (this *CyeamBlogCrawler) Detail(resp *maodou.Response) (*models.Result, error) {
+	var err error
 	res := new(models.Result)
 	u, _ := url.Parse(resp.Url)
 	res.Id = u.Path
@@ -67,18 +52,12 @@ func (this *CyeamBlogCrawler) Detail(resp *maodou.Response) {
 	res.Tags = strings.Join(tags, " ")
 	_category := resp.Doc("#content > div.row-fluid.post-full > div > ul:nth-child(6) > li:nth-child(2) > a").Text()
 	res.Category = strings.Split(strings.TrimSpace(_category), " ")[0]
-	this.Result(res)
-}
-
-func (this *CyeamBlogCrawler) Result(result *models.Result) {
-	logger.Info(result.Title, result.Link)
-	//if result.Figure != "" {
-	//	Dao, err := dao.NewDao("duoshuo", `{"short_name":"cyeam","secret":"df66f048bd56cba5bf219b51766dec0d"}`)
-	//	if err != nil {
-	//		panic(err)
-	//	}
-	//	Dao.AddResult(result)
-	//}
+	date := resp.Doc("#content > div.row-fluid.post-full > div > div.date > span:nth-child(2)").Text()
+	res.CreateTime, err = time.Parse("02 January 2006", date)
+	if err != nil {
+		return res, err
+	}
+	return res, nil
 }
 
 func main() {
@@ -87,6 +66,6 @@ func main() {
 
 func NewCyeamBlogCrawler() *CyeamBlogCrawler {
 	c := new(CyeamBlogCrawler)
-	c.MaoDou.Init()
+	c.MaoDou.Init("http://blog.cyeam.com/#all.html")
 	return c
 }