ItslearningDL.go/main.go

package main

import (
	"flag"
	"fmt"
	"minie4/itslearningdl/itslearning"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/charmbracelet/log"
	"github.com/gosuri/uiprogress"
	"github.com/joho/godotenv"
	"github.com/muesli/termenv"
)

var max_workers = 10
var out_path = "./out/"
var logger *log.Logger

type FSResource struct {
	resource itslearning.Resource
	fsPath   string
}

func watchForChanges(orig []itslearning.Course, itsl itslearning.Itslearning) itslearning.Course {
	for {
		time.Sleep(2 * time.Second)
		logger.Info("Checking for changes")
		newCourses := itslearning.QueryCourseList(itsl)
		for i, course := range orig {
			if course.LastUpdatedUtc != newCourses[i].LastUpdatedUtc {
				logger.Info("Found change", "course", course.Title, "id", course.CourseId)
				return course
			}
		}
		logger.Debug("No changes")
	}
}

func main() {
	var doWatch = flag.Bool("w", false, "Watch for changes")
	flag.Parse()

	bars := uiprogress.New()
	bars.Start()

	logger = log.New(bars.Bypass())
	logger.SetColorProfile(termenv.ANSI256)
	logger.SetLevel(log.DebugLevel)

	err := godotenv.Load()
	if err != nil {
		logger.Warn("Error loading .env file")
	}

	user_agent := "com.itslearning.itslearningintapp 3.7.1 (HONOR BLN-L21 / Android 9)"
	username := os.Getenv("ITSLEARNING_USERNAME")
	password := os.Getenv("ITSLEARNING_PASSWORD")
	instance := os.Getenv("ITSLEARNING_INSTANCE")

	if username == "" || password == "" || instance == "" {
		logger.Fatal("Environment variables missing!")
	}

	itsl := itslearning.New(username, password, instance, user_agent)
	courses := itslearning.QueryCourseList(itsl)
	if *doWatch {
		logger.Info("Entering watch mode. Checking for changes every :2 seconds...")
		courses = []itslearning.Course{watchForChanges(courses, itsl)}
	}

	bar_course := bars.AddBar(len(courses)).AppendFunc(appendProgress)
	bar_course.PrependFunc(func(b *uiprogress.Bar) string {
		return "Total progress"
	})
	for _, course := range courses {
		logger.Info("Downloading course", "course", course.Title, "id", course.CourseId)

		var resouceList []FSResource
		bars.Bars = []*uiprogress.Bar{bar_course}
		bar := bars.AddBar(1)
		bar.PrependFunc(func(b *uiprogress.Bar) string {
			return "Fetching files"
		})
		res := itslearning.QueryCourseResources(itsl, course.CourseId)
		for _, resource := range res {
			var coursePath = sanitizePath(course.Title) + "/" + sanitizePath(resource.Title)

			if resource.ElementType == "Folder" {
				fetchResourcesRecursive(itsl, course, resource.ElementID, coursePath, &resouceList)
				logger.Info("Found folder", "element", resource.Title, "id", resource.ElementID)
			}

			if resource.ElementType == "LearningToolElement" {
				var fsPath = out_path + coursePath
				var fsResource = FSResource{resource: resource, fsPath: fsPath}
				resouceList = append(resouceList, fsResource)
				logger.Info("Found element", "element", resource.Title, "id", resource.ElementID)
			}
		}
		bar.Incr()

		if len(resouceList) < 1 {
			continue
		}

		var wg sync.WaitGroup
		current_tasks := 0
		bars.Bars = []*uiprogress.Bar{bar_course}
		bar = bars.AddBar(len(resouceList)).AppendFunc(appendProgress)
		bar.PrependFunc(func(b *uiprogress.Bar) string {
			return "Downloading files"
		})
		for _, fsResource := range resouceList {
			var resource = fsResource.resource
			var fsPath = fsResource.fsPath
			if _, err := os.Stat(fsPath); err == nil {
				logger.Info("Skipping element", "course", course.Title, "element", resource.Title, "id", resource.ElementID)
				continue
			}
			logger.Info("Downloading element", "course", course.Title, "element", resource.Title, "id", resource.ElementID)
			wg.Add(1)
			current_tasks++
			go func(resource itslearning.Resource) {
				defer wg.Done()
				itslearning.DownloadElement(itsl, resource.ElementID, fsPath)
				bar.Incr()
				current_tasks--
			}(resource)

			if current_tasks >= max_workers {
				logger.Debug("Waiting for running downloads to finish")
				wg.Wait()
			}
		}
		wg.Wait()
		bar.Set(100)
		bar = nil

		bar_course.Incr()
	}
}

func fetchResourcesRecursive(itsl itslearning.Itslearning, course itslearning.Course, folder_resource int, coursePath string, resList *[]FSResource) {
	res := itslearning.QueryFolderResources(itsl, course.CourseId, folder_resource)
	for _, resource := range res {
		var newCoursePath = coursePath + "/" + sanitizePath(resource.Title)

		if resource.ElementType == "Folder" {
			fetchResourcesRecursive(itsl, course, resource.ElementID, newCoursePath, resList)
			logger.Info("Found subfolder", "element", resource.Title, "id", resource.ElementID)
		}

		if resource.ElementType == "LearningToolElement" {
			var fsPath = out_path + newCoursePath
			var fsResource = FSResource{resource: resource, fsPath: fsPath}
			*resList = append(*resList, fsResource)
			logger.Info("Found element", "element", resource.Title, "id", resource.ElementID)
		}
	}
}

func sanitizePath(path string) string {
	// Remove leading slash of filepath
	var regex = regexp.MustCompile(`^\s*?/\s*`)
	var sanitizedWhitespace = regex.ReplaceAllString(path, "")

	// Replace invalid filename characters with underscore
	var invalidCharsRegex = regexp.MustCompile(`[<>:"/\|?*]`)
	return invalidCharsRegex.ReplaceAllString(sanitizedWhitespace, "_")
}

func appendProgress(b *uiprogress.Bar) string {
	return fmt.Sprintf("%d/%d (%.2f%%)", b.Current(), b.Total, (float32(b.Current())/float32(b.Total))*100.0)
}