commit 578155f1db7173e045075fed32ed634671d57806 Author: minie4 Date: Wed Dec 20 00:29:58 2023 +0100 🎉 Support basic downloading diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e499e0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +out/ \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6280aca --- /dev/null +++ b/go.mod @@ -0,0 +1,25 @@ +module minie4/itslearningdl + +go 1.21.5 + +require ( + github.com/anaskhan96/soup v1.2.5 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/lipgloss v0.9.1 // indirect + github.com/charmbracelet/log v0.3.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/schollz/progressbar/v3 v3.14.1 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..17bd8f2 --- /dev/null +++ b/go.sum @@ -0,0 +1,55 @@ +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= +github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= +github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw= +github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= +github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/itslearning/itslearning.go b/itslearning/itslearning.go new file mode 100644 index 0000000..2a384cb --- /dev/null +++ b/itslearning/itslearning.go @@ -0,0 +1,230 @@ +package itslearning + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/anaskhan96/soup" + "github.com/charmbracelet/log" +) + +type itslearning struct { + instance string + user_agent string + access_token string +} + +type Course struct { + CourseCode string `json:"CourseCode"` + CourseColor string `json:"CourseColor"` + CourseFillColor string `json:"CourseFillColor"` + CourseId int `json:"CourseId"` + FriendlyName *string `json:"FriendlyName"` + HasAdminPermissions bool `json:"HasAdminPermissions"` + HasStudentPermissions bool `json:"HasStudentPermissions"` + LastUpdatedUtc time.Time `json:"LastUpdatedUtc"` + NewBulletinsCount int `json:"NewBulletinsCount"` + NewNotificationsCount int `json:"NewNotificationsCount"` + Title string `json:"Title"` + Url string `json:"Url"` +} + +type Resource struct { + Title string `json:"Title"` + ElementID int `json:"ElementId"` + ElementType string `json:"ElementType"` + CourseID int `json:"CourseId"` + URL string `json:"Url"` + ContentURL string `json:"ContentUrl"` + IconURL string `json:"IconUrl"` + Active bool `json:"Active"` + LearningToolID int `json:"LearningToolId"` + AddElementURL any `json:"AddElementUrl"` + Homework bool `json:"Homework"` + Path string `json:"Path"` + LearningObjectID int `json:"LearningObjectId"` + LearningObjectInstanceID int `json:"LearningObjectInstanceId"` +} + +func doHTTPReq(req http.Request, err error, itsl itslearning, method string, target interface{}) error { + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("User-Agent", itsl.user_agent) + if err != nil { + log.Warn("Itslearning request failed!", "method", method, "error", err) + return err + } + + res, err := http.DefaultClient.Do(&req) + if err != nil { + log.Warn("Itslearning request failed!", "method", method, "error", err) + return err + } + + defer res.Body.Close() + return json.NewDecoder(res.Body).Decode(target) +} + +func RequestAuthtoken(itsl itslearning, username string, password string) string { + path := itsl.instance + "/restapi/oauth2/token" + payload := url.Values{ + "client_id": {"10ae9d30-1853-48ff-81cb-47b58a325685"}, + "grant_type": {"password"}, + "username": {username}, + "password": {password}, + } + + req, err := http.NewRequest(http.MethodPost, path, strings.NewReader(payload.Encode())) + data := new(map[string]interface{}) + err = doHTTPReq(*req, err, itsl, "Login", data) + return (*data)["access_token"].(string) +} + +func QueryCourseList(itsl itslearning) []Course { + path := itsl.instance + "/restapi/personal/courses/v2" + payload := url.Values{ + "access_token": {itsl.access_token}, + "pageIndex": {"0"}, + "pageSize": {"9999"}, + "filter": {"1"}, + } + + req, err := http.NewRequest(http.MethodGet, path, strings.NewReader(payload.Encode())) + data := new(struct{ EntityArray []Course }) + doHTTPReq(*req, err, itsl, "QueryCourseList", data) + return data.EntityArray +} + +func QueryCourseResources(itsl itslearning, courseID int) []Resource { + path := fmt.Sprintf("%s/restapi/personal/courses/%s/resources/v1", itsl.instance, strconv.Itoa(courseID)) + payload := url.Values{ + "access_token": {itsl.access_token}, + "pageIndex": {"0"}, + "pageSize": {"9999"}, + } + + req, err := http.NewRequest(http.MethodGet, path, strings.NewReader(payload.Encode())) + data := new(struct { + Resources struct { + EntityArray []Resource + } + }) + doHTTPReq(*req, err, itsl, "QueryCourseResources", data) + return data.Resources.EntityArray +} + +func QueryFolderResources(itsl itslearning, courseID int, folderElementID int) []Resource { + path := fmt.Sprintf("%s/restapi/personal/courses/%s/folders/%s/resources/v1", itsl.instance, strconv.Itoa(courseID), strconv.Itoa(folderElementID)) + payload := url.Values{ + "access_token": {itsl.access_token}, + "pageIndex": {"0"}, + "pageSize": {"9999"}, + } + + req, err := http.NewRequest(http.MethodGet, path, strings.NewReader(payload.Encode())) + data := new(struct { + Resources struct { + EntityArray []Resource + } + }) + doHTTPReq(*req, err, itsl, "QueryFolderResources", data) + return data.Resources.EntityArray +} + +func DownloadElement(itsl itslearning, elementID int, outfile string) { + // Request the SSO redirect URL + path := fmt.Sprintf("%s/restapi/personal/sso/url/v1", itsl.instance) + payload := url.Values{ + "access_token": {itsl.access_token}, + "url": {fmt.Sprintf("%s/LearningToolElement/ViewLearningToolElement.aspx?LearningToolElementId=%d", itsl.instance, elementID)}, + } + + req, err := http.NewRequest(http.MethodGet, path, nil) + req.URL.RawQuery = payload.Encode() + data := new(struct{ Url string }) + doHTTPReq(*req, err, itsl, "QueryFolderResources", data) + if err != nil { + log.Error("Error fetching SSO URL:", err) + return + } + + jar, _ := cookiejar.New(nil) + client := &http.Client{ + Jar: jar, + } + log.Debug("SSO URL", "data", data) + + // Make a request to the SSO redirect + req, err = http.NewRequest(http.MethodGet, data.Url, nil) + req.Header.Set("User-Agent", itsl.user_agent) + res, err := client.Do(req) + if err != nil { + log.Warn("File download failed!", "error", err) + return + } + html, err := io.ReadAll(res.Body) + + // Get the iframe URL + doc := soup.HTMLParse(string(html)) + iframe := doc.Find("iframe") + if iframe.Error != nil { + log.Error(iframe.Error) + return + } + + iframeSrc := iframe.Attrs()["src"] + + log.Debug("File iframe Src", "src", iframeSrc) + + req, err = http.NewRequest(http.MethodGet, iframeSrc, nil) + req.Header.Set("User-Agent", itsl.user_agent) + res, err = client.Do(req) + if err != nil { + log.Warn("File download failed!", "error", err) + return + } + html, err = io.ReadAll(res.Body) + + redirectedUrl := res.Request.URL.String() + u, _ := url.Parse(redirectedUrl) + q, _ := url.ParseQuery(u.RawQuery) + + payload = url.Values{ + "LearningObjectId": {q.Get("LearningObjectId")}, + "LearningObjectInstanceId": {q.Get("LearningObjectInstanceId")}, + } + resourceUrl := "https://resource.itslearning.com/Proxy/DownloadRedirect.ashx" + req, err = http.NewRequest(http.MethodGet, resourceUrl, nil) + req.URL.RawQuery = payload.Encode() + req.Header.Set("User-Agent", itsl.user_agent) + res, err = client.Do(req) + if err != nil { + log.Warn("File download failed!", "error", err) + return + } + + dir, _ := filepath.Split(outfile) + os.MkdirAll(dir, os.ModePerm) + out, err := os.Create(outfile) + if err != nil { + log.Error("Failed writing file!", "error", err) + } + defer out.Close() + defer res.Body.Close() + io.Copy(out, res.Body) +} + +func New(username string, password string, instance string, useragent string) itslearning { + log.Debug("Attempting to log in") + e := itslearning{instance, useragent, ""} + e.access_token = RequestAuthtoken(e, username, password) + return e +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..10170e7 --- /dev/null +++ b/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "minie4/itslearningdl/itslearning" + "os" + + "github.com/charmbracelet/log" + "github.com/joho/godotenv" +) + +func main() { + + // log.SetLevel(log.DebugLevel) + + err := godotenv.Load() + if err != nil { + log.Warn("Error loading .env file") + } + + user_agent := "com.itslearning.itslearningintapp 3.7.1 (HONOR BLN-L21 / Android 9)" + username := os.Getenv("ITSLEARNING_USERNAME") + password := os.Getenv("ITSLEARNING_PASSWORD") + instance := os.Getenv("ITSLEARNING_INSTANCE") + + if username == "" || password == "" || instance == "" { + log.Fatal("Environment variables missing!") + } + + itsl := itslearning.New(username, password, instance, user_agent) + + courses := itslearning.QueryCourseList(itsl) + for _, course := range courses { + log.Info("Downloading course", "course", course.Title, "id", course.CourseId) + res := itslearning.QueryCourseResources(itsl, course.CourseId) + + for _, resource := range res { + if resource.ElementType != "LearningToolElement" { + continue + } + log.Info("Downloading element", "element", resource.Title, "id", resource.ElementID) + itslearning.DownloadElement(itsl, resource.ElementID, "./out/"+course.Title+"/"+resource.Title) + } + } + +}