From e1489652e53a58be56560ef0173eb9231e347d9c Mon Sep 17 00:00:00 2001 From: minie4 Date: Wed, 20 Dec 2023 22:50:30 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Implement=20subfolder=20downloading?= =?UTF-8?q?=20and=20progress=20bar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 18 ++++--- go.sum | 23 +++++---- itslearning/itslearning.go | 18 +++---- main.go | 102 ++++++++++++++++++++++++++++++++----- 4 files changed, 121 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 6280aca..452ab9f 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,25 @@ module minie4/itslearningdl go 1.21.5 require ( - github.com/anaskhan96/soup v1.2.5 // indirect + github.com/anaskhan96/soup v1.2.5 + github.com/charmbracelet/log v0.3.1 + github.com/gosuri/uiprogress v0.0.1 + github.com/joho/godotenv v1.5.1 + github.com/muesli/termenv v0.15.2 +) + +require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/lipgloss v0.9.1 // indirect - github.com/charmbracelet/log v0.3.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/joho/godotenv v1.5.1 // indirect + github.com/gosuri/uilive v0.0.4 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/schollz/progressbar/v3 v3.14.1 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect - golang.org/x/text v0.3.0 // indirect + golang.org/x/text v0.3.8 // indirect ) diff --git a/go.sum b/go.sum index 17bd8f2..2b790b8 100644 --- a/go.sum +++ b/go.sum @@ -7,12 +7,16 @@ github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9 github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw= github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= +github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= +github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= +github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -20,22 +24,20 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= -github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -43,13 +45,12 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCT golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/itslearning/itslearning.go b/itslearning/itslearning.go index 2a384cb..6a27d53 100644 --- a/itslearning/itslearning.go +++ b/itslearning/itslearning.go @@ -17,7 +17,7 @@ import ( "github.com/charmbracelet/log" ) -type itslearning struct { +type Itslearning struct { instance string user_agent string access_token string @@ -55,7 +55,7 @@ type Resource struct { LearningObjectInstanceID int `json:"LearningObjectInstanceId"` } -func doHTTPReq(req http.Request, err error, itsl itslearning, method string, target interface{}) error { +func doHTTPReq(req http.Request, err error, itsl Itslearning, method string, target interface{}) error { req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("User-Agent", itsl.user_agent) if err != nil { @@ -73,7 +73,7 @@ func doHTTPReq(req http.Request, err error, itsl itslearning, method string, tar return json.NewDecoder(res.Body).Decode(target) } -func RequestAuthtoken(itsl itslearning, username string, password string) string { +func RequestAuthtoken(itsl Itslearning, username string, password string) string { path := itsl.instance + "/restapi/oauth2/token" payload := url.Values{ "client_id": {"10ae9d30-1853-48ff-81cb-47b58a325685"}, @@ -88,7 +88,7 @@ func RequestAuthtoken(itsl itslearning, username string, password string) string return (*data)["access_token"].(string) } -func QueryCourseList(itsl itslearning) []Course { +func QueryCourseList(itsl Itslearning) []Course { path := itsl.instance + "/restapi/personal/courses/v2" payload := url.Values{ "access_token": {itsl.access_token}, @@ -103,7 +103,7 @@ func QueryCourseList(itsl itslearning) []Course { return data.EntityArray } -func QueryCourseResources(itsl itslearning, courseID int) []Resource { +func QueryCourseResources(itsl Itslearning, courseID int) []Resource { path := fmt.Sprintf("%s/restapi/personal/courses/%s/resources/v1", itsl.instance, strconv.Itoa(courseID)) payload := url.Values{ "access_token": {itsl.access_token}, @@ -121,7 +121,7 @@ func QueryCourseResources(itsl itslearning, courseID int) []Resource { return data.Resources.EntityArray } -func QueryFolderResources(itsl itslearning, courseID int, folderElementID int) []Resource { +func QueryFolderResources(itsl Itslearning, courseID int, folderElementID int) []Resource { path := fmt.Sprintf("%s/restapi/personal/courses/%s/folders/%s/resources/v1", itsl.instance, strconv.Itoa(courseID), strconv.Itoa(folderElementID)) payload := url.Values{ "access_token": {itsl.access_token}, @@ -139,7 +139,7 @@ func QueryFolderResources(itsl itslearning, courseID int, folderElementID int) [ return data.Resources.EntityArray } -func DownloadElement(itsl itslearning, elementID int, outfile string) { +func DownloadElement(itsl Itslearning, elementID int, outfile string) { // Request the SSO redirect URL path := fmt.Sprintf("%s/restapi/personal/sso/url/v1", itsl.instance) payload := url.Values{ @@ -222,9 +222,9 @@ func DownloadElement(itsl itslearning, elementID int, outfile string) { io.Copy(out, res.Body) } -func New(username string, password string, instance string, useragent string) itslearning { +func New(username string, password string, instance string, useragent string) Itslearning { log.Debug("Attempting to log in") - e := itslearning{instance, useragent, ""} + e := Itslearning{instance, useragent, ""} e.access_token = RequestAuthtoken(e, username, password) return e } diff --git a/main.go b/main.go index 10170e7..37871d2 100644 --- a/main.go +++ b/main.go @@ -1,20 +1,31 @@ package main import ( + "fmt" "minie4/itslearningdl/itslearning" "os" + "sync" "github.com/charmbracelet/log" + "github.com/gosuri/uiprogress" "github.com/joho/godotenv" + "github.com/muesli/termenv" ) -func main() { +var max_workers = 10 +var logger *log.Logger - // log.SetLevel(log.DebugLevel) +func main() { + bars := uiprogress.New() + bars.Start() + + logger = log.New(bars.Bypass()) + logger.SetColorProfile(termenv.ANSI256) + logger.SetLevel(log.DebugLevel) err := godotenv.Load() if err != nil { - log.Warn("Error loading .env file") + logger.Warn("Error loading .env file") } user_agent := "com.itslearning.itslearningintapp 3.7.1 (HONOR BLN-L21 / Android 9)" @@ -23,23 +34,90 @@ func main() { instance := os.Getenv("ITSLEARNING_INSTANCE") if username == "" || password == "" || instance == "" { - log.Fatal("Environment variables missing!") + logger.Fatal("Environment variables missing!") } itsl := itslearning.New(username, password, instance, user_agent) - courses := itslearning.QueryCourseList(itsl) - for _, course := range courses { - log.Info("Downloading course", "course", course.Title, "id", course.CourseId) - res := itslearning.QueryCourseResources(itsl, course.CourseId) + bar_course := bars.AddBar(len(courses)).AppendFunc(appendProgress) + bar_course.PrependFunc(func(b *uiprogress.Bar) string { + return "Total progress" + }) + for _, course := range courses { + logger.Info("Downloading course", "course", course.Title, "id", course.CourseId) + + var resouceList []itslearning.Resource + bars.Bars = []*uiprogress.Bar{bar_course} + bar := bars.AddBar(1) + bar.PrependFunc(func(b *uiprogress.Bar) string { + return "Fetching files" + }) + res := itslearning.QueryCourseResources(itsl, course.CourseId) for _, resource := range res { - if resource.ElementType != "LearningToolElement" { - continue + if resource.ElementType == "Folder" { + fetchResourcesRecursive(itsl, course, resource.ElementID, &resouceList) + logger.Info("Found folder", "element", resource.Title, "id", resource.ElementID) + } + + if resource.ElementType == "LearningToolElement" { + resouceList = append(resouceList, resource) + logger.Info("Found element", "element", resource.Title, "id", resource.ElementID) } - log.Info("Downloading element", "element", resource.Title, "id", resource.ElementID) - itslearning.DownloadElement(itsl, resource.ElementID, "./out/"+course.Title+"/"+resource.Title) } + bar.Incr() + + if len(resouceList) < 1 { + continue + } + + var wg sync.WaitGroup + current_tasks := 0 + bars.Bars = []*uiprogress.Bar{bar_course} + bar = bars.AddBar(len(resouceList)).AppendFunc(appendProgress) + bar.PrependFunc(func(b *uiprogress.Bar) string { + return "Downloading files" + }) + for _, resource := range resouceList { + logger.Info("Downloading element", "element", resource.Title, "id", resource.ElementID) + wg.Add(1) + current_tasks++ + go func(resource itslearning.Resource) { + defer wg.Done() + bar.Incr() + itslearning.DownloadElement(itsl, resource.ElementID, "./out/"+course.Title+"/"+resource.Title) + current_tasks-- + }(resource) + + if current_tasks >= max_workers { + logger.Debug("Waiting for running downloads to finish") + wg.Wait() + } + } + wg.Wait() + bar.Set(100) + bar = nil + + bar_course.Incr() } } + +func fetchResourcesRecursive(itsl itslearning.Itslearning, course itslearning.Course, folder_resource int, resList *[]itslearning.Resource) { + res := itslearning.QueryFolderResources(itsl, course.CourseId, folder_resource) + for _, resource := range res { + if resource.ElementType == "Folder" { + fetchResourcesRecursive(itsl, course, resource.ElementID, resList) + logger.Info("Found subfolder", "element", resource.Title, "id", resource.ElementID) + } + + if resource.ElementType == "LearningToolElement" { + *resList = append(*resList, resource) + logger.Info("Found element", "element", resource.Title, "id", resource.ElementID) + } + } +} + +func appendProgress(b *uiprogress.Bar) string { + return fmt.Sprintf("%d/%d (%.2f%%)", b.Current(), b.Total, (float32(b.Current())/float32(b.Total))*100.0) +}