minie4/ItslearningDL.go
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🐛 Apparently folders on itslearning can include slashes

Browse Source 
- Don't use the path parameter provided by the itslearning API
- Use a local sanitized path variable
This commit is contained in:
minie4
2024-01-25 13:51:59 +01:00
parent c83229f3be
commit b18fdd5922
1 changed files with 32 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
main.go
View File

@ -14,8 +14,14 @@ import (
) )
var max_workers = 10 var max_workers = 10
var out_path = "./out/"
var logger *log.Logger var logger *log.Logger
type FSResource struct {
resource itslearning.Resource
fsPath string
}
func main() { func main() {
bars := uiprogress.New() bars := uiprogress.New()
bars.Start() bars.Start()
@ -48,7 +54,7 @@ func main() {
for _, course := range courses { for _, course := range courses {
logger.Info("Downloading course", "course", course.Title, "id", course.CourseId) logger.Info("Downloading course", "course", course.Title, "id", course.CourseId)
var resouceList []itslearning.Resource var resouceList []FSResource
bars.Bars = []*uiprogress.Bar{bar_course} bars.Bars = []*uiprogress.Bar{bar_course}
bar := bars.AddBar(1) bar := bars.AddBar(1)
bar.PrependFunc(func(b *uiprogress.Bar) string { bar.PrependFunc(func(b *uiprogress.Bar) string {
@ -56,13 +62,17 @@ func main() {
}) })
res := itslearning.QueryCourseResources(itsl, course.CourseId) res := itslearning.QueryCourseResources(itsl, course.CourseId)
for _, resource := range res { for _, resource := range res {
var coursePath = sanitizePath(course.Title) + "/" + sanitizePath(resource.Title)
if resource.ElementType == "Folder" { if resource.ElementType == "Folder" {
fetchResourcesRecursive(itsl, course, resource.ElementID, &resouceList) fetchResourcesRecursive(itsl, course, resource.ElementID, coursePath, &resouceList)
logger.Info("Found folder", "element", resource.Title, "id", resource.ElementID) logger.Info("Found folder", "element", resource.Title, "id", resource.ElementID)
} }
if resource.ElementType == "LearningToolElement" { if resource.ElementType == "LearningToolElement" {
resouceList = append(resouceList, resource) var fsPath = out_path + coursePath
var fsResource = FSResource{resource: resource, fsPath: fsPath}
resouceList = append(resouceList, fsResource)
logger.Info("Found element", "element", resource.Title, "id", resource.ElementID) logger.Info("Found element", "element", resource.Title, "id", resource.ElementID)
} }
} }
@ -79,9 +89,10 @@ func main() {
bar.PrependFunc(func(b *uiprogress.Bar) string { bar.PrependFunc(func(b *uiprogress.Bar) string {
return "Downloading files" return "Downloading files"
}) })
for _, resource := range resouceList { for _, fsResource := range resouceList {
var out_path = "./out" + sanitizePath(resource.Path) + "/" + resource.Title var resource = fsResource.resource
if _, err := os.Stat(out_path); err == nil { var fsPath = fsResource.fsPath
if _, err := os.Stat(fsPath); err == nil {
logger.Info("Skipping element", "course", course.Title, "element", resource.Title, "id", resource.ElementID) logger.Info("Skipping element", "course", course.Title, "element", resource.Title, "id", resource.ElementID)
continue continue
} }
@ -90,7 +101,7 @@ func main() {
current_tasks++ current_tasks++
go func(resource itslearning.Resource) { go func(resource itslearning.Resource) {
defer wg.Done() defer wg.Done()
itslearning.DownloadElement(itsl, resource.ElementID, out_path) itslearning.DownloadElement(itsl, resource.ElementID, fsPath)
bar.Incr() bar.Incr()
current_tasks-- current_tasks--
}(resource) }(resource)
@ -108,24 +119,33 @@ func main() {
} }
} }
func fetchResourcesRecursive(itsl itslearning.Itslearning, course itslearning.Course, folder_resource int, resList *[]itslearning.Resource) { func fetchResourcesRecursive(itsl itslearning.Itslearning, course itslearning.Course, folder_resource int, coursePath string, resList *[]FSResource) {
res := itslearning.QueryFolderResources(itsl, course.CourseId, folder_resource) res := itslearning.QueryFolderResources(itsl, course.CourseId, folder_resource)
for _, resource := range res { for _, resource := range res {
var newCoursePath = coursePath + "/" + sanitizePath(resource.Title)
if resource.ElementType == "Folder" { if resource.ElementType == "Folder" {
fetchResourcesRecursive(itsl, course, resource.ElementID, resList) fetchResourcesRecursive(itsl, course, resource.ElementID, newCoursePath, resList)
logger.Info("Found subfolder", "element", resource.Title, "id", resource.ElementID) logger.Info("Found subfolder", "element", resource.Title, "id", resource.ElementID)
} }
if resource.ElementType == "LearningToolElement" { if resource.ElementType == "LearningToolElement" {
*resList = append(*resList, resource) var fsPath = out_path + newCoursePath
var fsResource = FSResource{resource: resource, fsPath: fsPath}
*resList = append(*resList, fsResource)
logger.Info("Found element", "element", resource.Title, "id", resource.ElementID) logger.Info("Found element", "element", resource.Title, "id", resource.ElementID)
} }
} }
} }
func sanitizePath(path string) string { func sanitizePath(path string) string {
var regex = regexp.MustCompile(`\s*?/\s*`) // Remove leading slash of filepath
return regex.ReplaceAllString(path, "/") var regex = regexp.MustCompile(`^\s*?/\s*`)
var sanitizedWhitespace = regex.ReplaceAllString(path, "")
// Replace invalid filename characters with underscore
var invalidCharsRegex = regexp.MustCompile(`[<>:"/\|?*]`)
return invalidCharsRegex.ReplaceAllString(sanitizedWhitespace, "_")
} }
func appendProgress(b *uiprogress.Bar) string { func appendProgress(b *uiprogress.Bar) string {