diff --git a/.env.template b/.env.template index d077973..db8b09b 100644 --- a/.env.template +++ b/.env.template @@ -3,8 +3,6 @@ LOGIN_NETID= LOGIN_PASSWORD= LOGIN_ASTRA_USERNAME= LOGIN_ASTRA_PASSWORD= -#Login to https://east.mymazevo.com/main-home then go to https://east.mymazevo.com/api/tenantsettings/GetApiKey -MAZEVO_API_KEY= #Academic Calendars GOOGLE_GENAI_USE_VERTEXAI= GOOGLE_CLOUD_PROJECT= diff --git a/parser/mazevoParser.go b/parser/mazevoParser.go index a4cca30..06b2646 100644 --- a/parser/mazevoParser.go +++ b/parser/mazevoParser.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "github.com/UTDNebula/api-tools/utils" @@ -23,60 +24,73 @@ type SourceData struct { // ParseMazevo reads Mazevo scrape output and emits normalized multi-building event JSON. func ParseMazevo(inDir string, outDir string) { - - mazevoFile, err := os.ReadFile(inDir + "/mazevoScraped.json") - if err != nil { - panic(err) - } - - var rawData SourceData - err = json.Unmarshal(mazevoFile, &rawData) + inDir = filepath.Join(inDir, "mazevo") + dir, err := os.ReadDir(inDir) if err != nil { - panic(err) + return } multiBuildingMap := make(map[string]map[string]map[string][]schema.MazevoEvent) - for _, rawEvent := range rawData.Bookings { - datePtr := utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]) - if datePtr == nil { + for _, file := range dir { + // Skip subdirectories + if file.IsDir() { continue } - date := (*datePtr)[:10] - building := utils.ConvertFromInterface[string](rawEvent["buildingDescription"]) - room := utils.ConvertFromInterface[string](rawEvent["roomDescription"]) - event := schema.MazevoEvent{ - EventName: utils.ConvertFromInterface[string](rawEvent["eventName"]), - OrganizationName: utils.ConvertFromInterface[string](rawEvent["organizationName"]), - ContactName: utils.ConvertFromInterface[string](rawEvent["contactName"]), - SetupMinutes: utils.ConvertFromInterface[float64](rawEvent["setupMinutes"]), - DateTimeStart: utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]), - DateTimeEnd: utils.ConvertFromInterface[string](rawEvent["dateTimeEnd"]), - TeardownMinutes: utils.ConvertFromInterface[float64](rawEvent["teardownMinutes"]), - StatusDescription: utils.ConvertFromInterface[string](rawEvent["statusDescription"]), - StatusColor: utils.ConvertFromInterface[string](rawEvent["statusColor"]), + + filePath := filepath.Join(inDir, file.Name()) + fileBytes, err := os.ReadFile(filePath) + if err != nil { + panic(err) } - if building == nil || room == nil || *(building) == "" || *(room) == "" { - continue + var rawData SourceData + err = json.Unmarshal(fileBytes, &rawData) + if err != nil { + panic(err) } - *building = strings.TrimSpace(*building) - for key, value := range buildingRenames { - if *building == key { - *building = value + + for _, rawEvent := range rawData.Bookings { + datePtr := utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]) + if datePtr == nil { + continue } - if strings.HasPrefix(*room, value+" ") { - *room = strings.TrimPrefix(*room, value+" ") + date := (*datePtr)[:10] + building := utils.ConvertFromInterface[string](rawEvent["buildingDescription"]) + room := utils.ConvertFromInterface[string](rawEvent["roomDescription"]) + event := schema.MazevoEvent{ + EventName: utils.ConvertFromInterface[string](rawEvent["eventName"]), + OrganizationName: utils.ConvertFromInterface[string](rawEvent["organizationName"]), + ContactName: utils.ConvertFromInterface[string](rawEvent["contactName"]), + SetupMinutes: utils.ConvertFromInterface[float64](rawEvent["setupMinutes"]), + DateTimeStart: utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]), + DateTimeEnd: utils.ConvertFromInterface[string](rawEvent["dateTimeEnd"]), + TeardownMinutes: utils.ConvertFromInterface[float64](rawEvent["teardownMinutes"]), + StatusDescription: utils.ConvertFromInterface[string](rawEvent["statusDescription"]), + StatusColor: utils.ConvertFromInterface[string](rawEvent["statusColor"]), } - } - if _, exists := multiBuildingMap[date]; !exists { - multiBuildingMap[date] = make(map[string]map[string][]schema.MazevoEvent) - } - if _, exists := multiBuildingMap[date][*building]; !exists { - multiBuildingMap[date][*building] = make(map[string][]schema.MazevoEvent) + if building == nil || room == nil || *(building) == "" || *(room) == "" { + continue + } + *building = strings.TrimSpace(*building) + for key, value := range buildingRenames { + if *building == key { + *building = value + } + if after, ok := strings.CutPrefix(*room, value+" "); ok { + *room = after + } + } + + if _, exists := multiBuildingMap[date]; !exists { + multiBuildingMap[date] = make(map[string]map[string][]schema.MazevoEvent) + } + if _, exists := multiBuildingMap[date][*building]; !exists { + multiBuildingMap[date][*building] = make(map[string][]schema.MazevoEvent) + } + multiBuildingMap[date][*building][*room] = append(multiBuildingMap[date][*building][*room], event) } - multiBuildingMap[date][*building][*room] = append(multiBuildingMap[date][*building][*room], event) } var result []schema.MultiBuildingEvents[schema.MazevoEvent] @@ -104,5 +118,8 @@ func ParseMazevo(inDir string, outDir string) { log.Print("Parsed Mazevo!") - utils.WriteJSON(fmt.Sprintf("%s/mazevo.json", outDir), result) + err = utils.WriteJSON(fmt.Sprintf("%s/mazevo.json", outDir), result) + if err != nil { + log.Panic(err) + } } diff --git a/scrapers/mazevo.go b/scrapers/mazevo.go index fb50df0..8910679 100644 --- a/scrapers/mazevo.go +++ b/scrapers/mazevo.go @@ -5,85 +5,129 @@ package scrapers import ( - "bytes" + "context" + "encoding/base64" "encoding/json" "fmt" - "io" "log" - "net/http" "os" + "path/filepath" "time" "github.com/UTDNebula/api-tools/utils" + "github.com/chromedp/cdproto/network" + + "github.com/chromedp/chromedp" ) // ScrapeMazevo pulls Mazevo calendar events via the public API and stores the raw response. func ScrapeMazevo(outDir string) { - apikey, err := utils.GetEnv("MAZEVO_API_KEY") - if err != nil { - panic(err) - } - // Make output folder - err = os.MkdirAll(outDir, 0777) + outDir = filepath.Join(outDir, "Mazevo") + err := os.MkdirAll(outDir, 0777) if err != nil { - panic(err) + log.Panic(err) } - // Init http client - tr := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 30 * time.Second, - DisableCompression: true, - } - cli := &http.Client{Transport: tr} - - // Start on previous date to make sure we have today's data, regardless of what timezone the scraper is in - date := time.Now() - startDate := date.Add(time.Hour * -24).Format(time.RFC3339) - endDate := date.Add(time.Hour * 24 * 365).Format(time.RFC3339) - - // Request events - url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents" - requestBodyMap := map[string]string{ - "apiKey": apikey, - "end": endDate, - "start": startDate, - } - requestBodyBytes, _ := json.Marshal(requestBodyMap) - requestBody := bytes.NewBuffer(requestBodyBytes) - req, err := http.NewRequest("POST", url, requestBody) - if err != nil { - panic(err) - } - req.Header = http.Header{ - "Content-type": {"application/json"}, - "Accept": {"application/json"}, - } - res, err := cli.Do(req) - if err != nil { - panic(err) - } - if res.StatusCode != 200 { - log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - } - res.Body.Close() - stringBody := string(body) + ctx, cancel := utils.InitChromeDp() + defer cancel() + var reqID network.RequestID // ID for requests + var eventsStart time.Time // Start time of events request + var eventsEnd time.Time // End time of events request - log.Printf("Scraped Mazevo up to %s!", endDate) + isPending := false + receivedChan := make(chan struct{}) - // Write event data to output file - fptr, err := os.Create(fmt.Sprintf("%s/mazevoScraped.json", outDir)) - if err != nil { - panic(err) + chromedp.ListenTarget(ctx, func(ev any) { + switch ev := ev.(type) { + case *network.EventRequestWillBeSent: + // Get GetEvents Request Start and End Times (ISO 8601) + if ev.Request.URL == "https://east.mymazevo.com/api/PublicCalendar/GetEvents" { + rawPostData := ev.Request.PostDataEntries[0].Bytes + decodedPostData, err := base64.StdEncoding.DecodeString(rawPostData) + if err != nil { + log.Panic(err) + } + var data map[string]any + + err = json.Unmarshal(decodedPostData, &data) + if err != nil { + log.Panic(err) + } + eventsStart, err = time.Parse(time.RFC3339, data["start"].(string)) + if err != nil { + log.Panic(err) + } + eventsEnd, err = time.Parse(time.RFC3339, data["end"].(string)) + if err != nil { + log.Panic(err) + } + + // Check if end is 1 month after start + if eventsEnd.After(eventsStart.AddDate(0, 1, -1)) { + isPending = true + } + } + case *network.EventResponseReceived: + // Once Response is received, record the RequestID + if isPending && ev.Response.URL == "https://east.mymazevo.com/api/PublicCalendar/GetEvents" { + reqID = ev.RequestID + } + case *network.EventLoadingFinished: + // Signal that response is finished loading + if isPending && ev.RequestID == reqID { + isPending = false + receivedChan <- struct{}{} + } + } + + }) + + scrapeLoop := func(ctx context.Context) error { + // Wait until events have been received + <-receivedChan + + // Read Response (JSON) + bodyBytes, err := network.GetResponseBody(reqID).Do(ctx) + if err != nil { + return fmt.Errorf("failed to get body: %w", err) + } + log.Printf("Scraped Mazevo from %s to %s!", eventsStart.Format(time.DateTime), eventsEnd.Format(time.DateTime)) + + // Write event data to output file + fptr, err := os.Create(fmt.Sprintf("%s/%s.json", outDir, eventsStart.Format("2006-01"))) + if err != nil { + log.Panic(err) + } + _, err = fptr.Write(bodyBytes) + if err != nil { + log.Panic(err) + } + + // Click next month + err = chromedp.Click("[aria-label=\"Move to Next Month\"]", chromedp.NodeVisible).Do(ctx) + if err != nil { + log.Panic(err) + } + return nil } - _, err = fptr.Write([]byte(stringBody)) + + _, err = chromedp.RunResponse(ctx, + chromedp.Navigate("https://east.mymazevo.com/calendar/4219c6df695c03860350ea213837fe59"), + chromedp.Sleep(5*time.Second), + chromedp.Click("input#displayMonth", chromedp.NodeVisible), + chromedp.ActionFunc(func(ctx context.Context) error { + for range 12 { // Scrape 12 months + err := scrapeLoop(ctx) + if err != nil { + return err + } + } + return nil + }), + ) if err != nil { - panic(err) + log.Panic(err) } - fptr.Close() + }