Skip to content

Commit 9f3796a

Browse files
spf13spf13
committed
Read/reread individual source content files
next is incremental conversion
1 parent ca6ca4f commit 9f3796a

File tree

7 files changed

+248
-76
lines changed

7 files changed

+248
-76
lines changed

‎commands/hugo.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,7 @@ func NewWatcher(port int) error {
594594

595595
for _, ev := range evs {
596596
ext := filepath.Ext(ev.Name)
597-
istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___")|| strings.HasSuffix(ext, "jb_bak___")
597+
istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___") || strings.HasSuffix(ext, "jb_bak___")
598598
if istemp {
599599
continue
600600
}
@@ -703,7 +703,7 @@ func NewWatcher(port int) error {
703703
fmt.Println(time.Now().Format(layout))
704704
//TODO here
705705

706-
// utils.CheckErr(buildSite(true))
706+
// utils.CheckErr(buildSite(true))
707707
rebuildSite(dynamicFilesChanged)
708708

709709
if !BuildWatch && !viper.GetBool("DisableLiveReload") {

‎docs/content/meta/roadmap.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ In no particular order, here is what we are working on:
1919
* Import from other website systems
2020
* from Drupal (See https://bitbucket.org/rickb777/drupal2hugo by Rick Beton (@rickb777))
2121
* from WordPress (See [#100][], especially https://github.com/SchumacherFM/wordpress-to-hugo-exporter by Cyrill Schumacher (@SchumacherFM), but volunteers are needed to make it work with latest versions of WordPress.)
22-
* from Jekyll (See [#101][])
2322
* An interactive web based editor (See http://discuss.gohugo.io/t/web-based-editor/155)
2423
* Additional [themes](https://github.com/spf13/hugoThemes) (always on-going, contributions welcome!)
2524
* Dynamic image resizing via shortcodes

‎hugolib/handler_page.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type basicPageHandler Handle
3232

3333
func (b basicPageHandler) Read(f *source.File, s *Site) HandledResult {
3434
page, err := NewPage(f.Path())
35+
3536
if err != nil {
3637
return HandledResult{file: f, err: err}
3738
}

‎hugolib/page.go

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -48,20 +48,19 @@ var (
4848
)
4949

5050
type Page struct {
51-
Params map[string]interface{}
52-
Content template.HTML
53-
Summary template.HTML
54-
Aliases []string
55-
Status string
56-
Images []Image
57-
Videos []Video
58-
TableOfContents template.HTML
59-
Truncated bool
60-
Draft bool
61-
PublishDate time.Time
62-
Tmpl tpl.Template
63-
Markup string
64-
51+
Params map[string]interface{}
52+
Content template.HTML
53+
Summary template.HTML
54+
Aliases []string
55+
Status string
56+
Images []Image
57+
Videos []Video
58+
TableOfContents template.HTML
59+
Truncated bool
60+
Draft bool
61+
PublishDate time.Time
62+
Tmpl tpl.Template
63+
Markup string
6564
extension string
6665
contentType string
6766
renderable bool
@@ -77,13 +76,13 @@ type Page struct {
7776
plainSecondaryInit sync.Once
7877
renderingConfig *helpers.Blackfriday
7978
renderingConfigInit sync.Once
79+
pageMenus PageMenus
80+
pageMenusInit sync.Once
81+
isCJKLanguage bool
8082
PageMeta
8183
Source
8284
Position `json:"-"`
8385
Node
84-
pageMenus PageMenus
85-
pageMenusInit sync.Once
86-
isCJKLanguage bool
8786
}
8887

8988
type Source struct {
@@ -106,6 +105,42 @@ type Position struct {
106105
}
107106

108107
type Pages []*Page
108+
//
109+
//func (ps Pages) Replace(page *Page) {
110+
// if i := ps.FindPagePos(page); i >= 0 {
111+
// ps[i] = page
112+
// }
113+
//}
114+
115+
//func (ps Pages) FindPageByFilePath(inPath string) *Page {
116+
// for _, x := range ps {
117+
// if x.Source.LogicalName() == inPath {
118+
// return x
119+
// }
120+
// }
121+
// return nil
122+
//}
123+
124+
// FindPagePos Given a page, it will find the position in Pages
125+
// will return -1 if not found
126+
func (ps Pages) FindPagePos(page *Page) int {
127+
for i, x := range ps {
128+
if x.Source.LogicalName() == page.Source.LogicalName() {
129+
return i
130+
}
131+
}
132+
return -1
133+
}
134+
135+
// FindPage Given a page, it will return the page in Pages
136+
// will return nil if not found
137+
//func (ps Pages) FindPage(page *Page) *Page {
138+
// if i := ps.FindPagePos(page); i >= 0 {
139+
// return ps[i]
140+
// }
141+
//
142+
// return nil
143+
//}
109144

110145
func (p *Page) Plain() string {
111146
p.initPlain()

‎hugolib/site.go

Lines changed: 145 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
451451
}
452452
}
453453

454-
455454
if len(tmplChanged) > 0 {
456455
s.prepTemplates()
457456
s.Tmpl.PrintErrors()
@@ -462,10 +461,41 @@ func (s *Site) ReBuild(changed map[string]bool) error {
462461
s.ReadDataFromSourceFS()
463462
}
464463

465-
if len (sourceChanged) > 0 {
466-
if err = s.CreatePages(); err != nil {
467-
return err
464+
if len(sourceChanged) > 0 {
465+
466+
results := make(chan HandledResult)
467+
filechan := make(chan *source.File)
468+
errs := make(chan error)
469+
wg := &sync.WaitGroup{}
470+
471+
wg.Add(2)
472+
for i := 0; i < 2; i++ {
473+
go sourceReader(s, filechan, results, wg)
474+
}
475+
476+
go incrementalReadCollator(s, results, errs)
477+
478+
for _, x := range sourceChanged {
479+
file, err := s.ReReadFile(x)
480+
if err != nil {
481+
errs <- err
482+
}
483+
484+
filechan <- file
468485
}
486+
487+
close(filechan)
488+
wg.Wait()
489+
close(results)
490+
491+
s.timerStep("read pages from source")
492+
493+
//renderErrs := <-s.ConvertSource()
494+
s.timerStep("convert source")
495+
// TODO(spf13) port this
496+
497+
fmt.Errorf("%s", errs)
498+
469499
s.setupPrevNext()
470500
if err = s.BuildSiteMeta(); err != nil {
471501
return err
@@ -497,7 +527,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
497527
return nil
498528
}
499529

500-
501530
func (s *Site) Analyze() error {
502531
if err := s.Process(); err != nil {
503532
return err
@@ -764,6 +793,47 @@ type pageResult struct {
764793
err error
765794
}
766795

796+
// ReReadFile resets file to be read from disk again
797+
func (s *Site) ReReadFile(absFilePath string) (*source.File, error) {
798+
fmt.Println("rereading", absFilePath)
799+
var file *source.File
800+
801+
reader, err := source.NewLazyFileReader(absFilePath)
802+
if err != nil {
803+
return nil, err
804+
}
805+
fmt.Println(s.absDataDir())
806+
807+
file, err = source.NewFileFromAbs(s.absContentDir(), absFilePath, reader)
808+
809+
fmt.Println("file created", file.Path())
810+
811+
if err != nil {
812+
return nil, err
813+
}
814+
815+
// maybe none of this rest needs to be here.
816+
// replaced := false
817+
818+
// fmt.Println(len(s.Files))
819+
820+
// for i, x := range s.Files {
821+
// fmt.Println(x)
822+
// fmt.Println("*** COMPARING:")
823+
// fmt.Println(" ", x.LogicalName())
824+
// fmt.Println(" ", absFilePath)
825+
// if x.LogicalName() == absFilePath {
826+
// s.Files[i] = file
827+
// replaced = true
828+
// }
829+
// }
830+
831+
// if !replaced {
832+
// s.Files = append(s.Files, file)
833+
// }
834+
return file, nil
835+
}
836+
767837
func (s *Site) ReadPagesFromSource() chan error {
768838
if s.Source == nil {
769839
panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
@@ -856,12 +926,17 @@ func (s *Site) CreatePages() error {
856926
func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResult, wg *sync.WaitGroup) {
857927
defer wg.Done()
858928
for file := range files {
859-
h := NewMetaHandler(file.Extension())
860-
if h != nil {
861-
h.Read(file, s, results)
862-
} else {
863-
jww.ERROR.Println("Unsupported File Type", file.Path())
864-
}
929+
fmt.Println("reading", file.Path())
930+
readSourceFile(s, file, results)
931+
}
932+
}
933+
934+
func readSourceFile(s *Site, file *source.File, results chan<- HandledResult) {
935+
h := NewMetaHandler(file.Extension())
936+
if h != nil {
937+
h.Read(file, s, results)
938+
} else {
939+
jww.ERROR.Println("Unsupported File Type", file.Path())
865940
}
866941
}
867942

@@ -905,7 +980,41 @@ func converterCollator(s *Site, results <-chan HandledResult, errs chan<- error)
905980
errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
906981
}
907982

908-
func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
983+
func (s *Site) AddPage(page *Page) {
984+
if page.ShouldBuild() {
985+
s.Pages = append(s.Pages, page)
986+
}
987+
988+
if page.IsDraft() {
989+
s.draftCount++
990+
}
991+
992+
if page.IsFuture() {
993+
s.futureCount++
994+
}
995+
}
996+
997+
func (s *Site) RemovePage(page *Page) {
998+
if i := s.Pages.FindPagePos(page); i >= 0 {
999+
if page.IsDraft() {
1000+
s.draftCount--
1001+
}
1002+
1003+
if page.IsFuture() {
1004+
s.futureCount--
1005+
}
1006+
1007+
s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
1008+
}
1009+
}
1010+
1011+
func (s *Site) ReplacePage(page *Page) {
1012+
// will find existing page that matches filepath and remove it
1013+
s.RemovePage(page)
1014+
s.AddPage(page)
1015+
}
1016+
1017+
func incrementalReadCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
9091018
errMsgs := []string{}
9101019
for r := range results {
9111020
if r.err != nil {
@@ -915,19 +1024,34 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
9151024

9161025
// !page == file
9171026
if r.page == nil {
1027+
// TODO(spf13): Make this incremental as well
9181028
s.Files = append(s.Files, r.file)
9191029
} else {
920-
if r.page.ShouldBuild() {
921-
s.Pages = append(s.Pages, r.page)
922-
}
1030+
s.ReplacePage(r.page)
1031+
}
1032+
}
9231033

924-
if r.page.IsDraft() {
925-
s.draftCount++
926-
}
1034+
s.Pages.Sort()
1035+
if len(errMsgs) == 0 {
1036+
errs <- nil
1037+
return
1038+
}
1039+
errs <- fmt.Errorf("Errors reading pages: %s", strings.Join(errMsgs, "\n"))
1040+
}
9271041

928-
if r.page.IsFuture() {
929-
s.futureCount++
930-
}
1042+
func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
1043+
errMsgs := []string{}
1044+
for r := range results {
1045+
if r.err != nil {
1046+
errMsgs = append(errMsgs, r.Error())
1047+
continue
1048+
}
1049+
1050+
// !page == file
1051+
if r.page == nil {
1052+
s.Files = append(s.Files, r.file)
1053+
} else {
1054+
s.AddPage(r.page)
9311055
}
9321056
}
9331057

0 commit comments

Comments
 (0)