Mazevo api key fix

.env.template

@@ -3,8 +3,6 @@ LOGIN_NETID=
 LOGIN_PASSWORD=
 LOGIN_ASTRA_USERNAME=
 LOGIN_ASTRA_PASSWORD=
-#Login to https://east.mymazevo.com/main-home then go to https://east.mymazevo.com/api/tenantsettings/GetApiKey
-MAZEVO_API_KEY=
 #Academic Calendars
 GOOGLE_GENAI_USE_VERTEXAI=
 GOOGLE_CLOUD_PROJECT=

parser/mazevoParser.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/UTDNebula/api-tools/utils"
@@ -23,60 +24,73 @@ type SourceData struct {
 
 // ParseMazevo reads Mazevo scrape output and emits normalized multi-building event JSON.
 func ParseMazevo(inDir string, outDir string) {
-
-	mazevoFile, err := os.ReadFile(inDir + "/mazevoScraped.json")
-	if err != nil {
-		panic(err)
-	}
-
-	var rawData SourceData
-	err = json.Unmarshal(mazevoFile, &rawData)
+	inDir = filepath.Join(inDir, "mazevo")
+	dir, err := os.ReadDir(inDir)
 	if err != nil {
-		panic(err)
+		return
 	}
 
 	multiBuildingMap := make(map[string]map[string]map[string][]schema.MazevoEvent)
 
-	for _, rawEvent := range rawData.Bookings {
-		datePtr := utils.ConvertFromInterface[string](rawEvent["dateTimeStart"])
-		if datePtr == nil {
+	for _, file := range dir {
+		// Skip subdirectories
+		if file.IsDir() {
 			continue
 		}
-		date := (*datePtr)[:10]
-		building := utils.ConvertFromInterface[string](rawEvent["buildingDescription"])
-		room := utils.ConvertFromInterface[string](rawEvent["roomDescription"])
-		event := schema.MazevoEvent{
-			EventName:         utils.ConvertFromInterface[string](rawEvent["eventName"]),
-			OrganizationName:  utils.ConvertFromInterface[string](rawEvent["organizationName"]),
-			ContactName:       utils.ConvertFromInterface[string](rawEvent["contactName"]),
-			SetupMinutes:      utils.ConvertFromInterface[float64](rawEvent["setupMinutes"]),
-			DateTimeStart:     utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]),
-			DateTimeEnd:       utils.ConvertFromInterface[string](rawEvent["dateTimeEnd"]),
-			TeardownMinutes:   utils.ConvertFromInterface[float64](rawEvent["teardownMinutes"]),
-			StatusDescription: utils.ConvertFromInterface[string](rawEvent["statusDescription"]),
-			StatusColor:       utils.ConvertFromInterface[string](rawEvent["statusColor"]),
+
+		filePath := filepath.Join(inDir, file.Name())
+		fileBytes, err := os.ReadFile(filePath)
+		if err != nil {
+			panic(err)
 		}
 
-		if building == nil || room == nil || *(building) == "" || *(room) == "" {
-			continue
+		var rawData SourceData
+		err = json.Unmarshal(fileBytes, &rawData)
+		if err != nil {
+			panic(err)
 		}
-		*building = strings.TrimSpace(*building)
-		for key, value := range buildingRenames {
-			if *building == key {
-				*building = value
+
+		for _, rawEvent := range rawData.Bookings {
+			datePtr := utils.ConvertFromInterface[string](rawEvent["dateTimeStart"])
+			if datePtr == nil {
+				continue
 			}
-			if strings.HasPrefix(*room, value+" ") {
-				*room = strings.TrimPrefix(*room, value+" ")
+			date := (*datePtr)[:10]
+			building := utils.ConvertFromInterface[string](rawEvent["buildingDescription"])
+			room := utils.ConvertFromInterface[string](rawEvent["roomDescription"])
+			event := schema.MazevoEvent{
+				EventName:         utils.ConvertFromInterface[string](rawEvent["eventName"]),
+				OrganizationName:  utils.ConvertFromInterface[string](rawEvent["organizationName"]),
+				ContactName:       utils.ConvertFromInterface[string](rawEvent["contactName"]),
+				SetupMinutes:      utils.ConvertFromInterface[float64](rawEvent["setupMinutes"]),
+				DateTimeStart:     utils.ConvertFromInterface[string](rawEvent["dateTimeStart"]),
+				DateTimeEnd:       utils.ConvertFromInterface[string](rawEvent["dateTimeEnd"]),
+				TeardownMinutes:   utils.ConvertFromInterface[float64](rawEvent["teardownMinutes"]),
+				StatusDescription: utils.ConvertFromInterface[string](rawEvent["statusDescription"]),
+				StatusColor:       utils.ConvertFromInterface[string](rawEvent["statusColor"]),
 			}
-		}
 
-		if _, exists := multiBuildingMap[date]; !exists {
-			multiBuildingMap[date] = make(map[string]map[string][]schema.MazevoEvent)
-		}
-		if _, exists := multiBuildingMap[date][*building]; !exists {
-			multiBuildingMap[date][*building] = make(map[string][]schema.MazevoEvent)
+			if building == nil || room == nil || *(building) == "" || *(room) == "" {
+				continue
+			}
+			*building = strings.TrimSpace(*building)
+			for key, value := range buildingRenames {
+				if *building == key {
+					*building = value
+				}
+				if after, ok := strings.CutPrefix(*room, value+" "); ok {
+					*room = after
+				}
+			}
+
+			if _, exists := multiBuildingMap[date]; !exists {
+				multiBuildingMap[date] = make(map[string]map[string][]schema.MazevoEvent)
+			}
+			if _, exists := multiBuildingMap[date][*building]; !exists {
+				multiBuildingMap[date][*building] = make(map[string][]schema.MazevoEvent)
+			}
+			multiBuildingMap[date][*building][*room] = append(multiBuildingMap[date][*building][*room], event)
 		}
-		multiBuildingMap[date][*building][*room] = append(multiBuildingMap[date][*building][*room], event)
 	}
 
 	var result []schema.MultiBuildingEvents[schema.MazevoEvent]
@@ -104,5 +118,8 @@ func ParseMazevo(inDir string, outDir string) {
 
 	log.Print("Parsed Mazevo!")
 
-	utils.WriteJSON(fmt.Sprintf("%s/mazevo.json", outDir), result)
+	err = utils.WriteJSON(fmt.Sprintf("%s/mazevo.json", outDir), result)
+	if err != nil {
+		log.Panic(err)
+	}
 }

scrapers/mazevo.go

@@ -5,85 +5,129 @@
 package scrapers
 
 import (
-	"bytes"
+	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io"
 	"log"
-	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/UTDNebula/api-tools/utils"
+	"github.com/chromedp/cdproto/network"
+
+	"github.com/chromedp/chromedp"
 )
 
 // ScrapeMazevo pulls Mazevo calendar events via the public API and stores the raw response.
 func ScrapeMazevo(outDir string) {
-	apikey, err := utils.GetEnv("MAZEVO_API_KEY")
-	if err != nil {
-		panic(err)
-	}
-
 	// Make output folder
-	err = os.MkdirAll(outDir, 0777)
+	outDir = filepath.Join(outDir, "Mazevo")
+	err := os.MkdirAll(outDir, 0777)
 	if err != nil {
-		panic(err)
+		log.Panic(err)
 	}
 
-	// Init http client
-	tr := &http.Transport{
-		MaxIdleConns:       10,
-		IdleConnTimeout:    30 * time.Second,
-		DisableCompression: true,
-	}
-	cli := &http.Client{Transport: tr}
-
-	// Start on previous date to make sure we have today's data, regardless of what timezone the scraper is in
-	date := time.Now()
-	startDate := date.Add(time.Hour * -24).Format(time.RFC3339)
-	endDate := date.Add(time.Hour * 24 * 365).Format(time.RFC3339)
-
-	// Request events
-	url := "https://east.mymazevo.com/api/PublicCalendar/GetCalendarEvents"
-	requestBodyMap := map[string]string{
-		"apiKey": apikey,
-		"end":    endDate,
-		"start":  startDate,
-	}
-	requestBodyBytes, _ := json.Marshal(requestBodyMap)
-	requestBody := bytes.NewBuffer(requestBodyBytes)
-	req, err := http.NewRequest("POST", url, requestBody)
-	if err != nil {
-		panic(err)
-	}
-	req.Header = http.Header{
-		"Content-type": {"application/json"},
-		"Accept":       {"application/json"},
-	}
-	res, err := cli.Do(req)
-	if err != nil {
-		panic(err)
-	}
-	if res.StatusCode != 200 {
-		log.Panicf("ERROR: Status was: %s\nIf the status is 404, you've likely been IP ratelimited!", res.Status)
-	}
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		panic(err)
-	}
-	res.Body.Close()
-	stringBody := string(body)
+	ctx, cancel := utils.InitChromeDp()
+	defer cancel()
+	var reqID network.RequestID // ID for requests
+	var eventsStart time.Time   // Start time of events request
+	var eventsEnd time.Time     // End time of events request
 
-	log.Printf("Scraped Mazevo up to %s!", endDate)
+	isPending := false
+	receivedChan := make(chan struct{})
 
-	// Write event data to output file
-	fptr, err := os.Create(fmt.Sprintf("%s/mazevoScraped.json", outDir))
-	if err != nil {
-		panic(err)
+	chromedp.ListenTarget(ctx, func(ev any) {
+		switch ev := ev.(type) {
+		case *network.EventRequestWillBeSent:
+			// Get GetEvents Request Start and End Times (ISO 8601)
+			if ev.Request.URL == "https://east.mymazevo.com/api/PublicCalendar/GetEvents" {
+				rawPostData := ev.Request.PostDataEntries[0].Bytes
+				decodedPostData, err := base64.StdEncoding.DecodeString(rawPostData)
+				if err != nil {
+					log.Panic(err)
+				}
+				var data map[string]any
+
+				err = json.Unmarshal(decodedPostData, &data)
+				if err != nil {
+					log.Panic(err)
+				}
+				eventsStart, err = time.Parse(time.RFC3339, data["start"].(string))
+				if err != nil {
+					log.Panic(err)
+				}
+				eventsEnd, err = time.Parse(time.RFC3339, data["end"].(string))
+				if err != nil {
+					log.Panic(err)
+				}
+
+				// Check if end is 1 month after start
+				if eventsEnd.After(eventsStart.AddDate(0, 1, -1)) {
+					isPending = true
+				}
+			}
+		case *network.EventResponseReceived:
+			// Once Response is received, record the RequestID
+			if isPending && ev.Response.URL == "https://east.mymazevo.com/api/PublicCalendar/GetEvents" {
+				reqID = ev.RequestID
+			}
+		case *network.EventLoadingFinished:
+			// Signal that response is finished loading
+			if isPending && ev.RequestID == reqID {
+				isPending = false
+				receivedChan <- struct{}{}
+			}
+		}
+
+	})
+
+	scrapeLoop := func(ctx context.Context) error {
+		// Wait until events have been received
+		<-receivedChan
+
+		// Read Response (JSON)
+		bodyBytes, err := network.GetResponseBody(reqID).Do(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to get body: %w", err)
+		}
+		log.Printf("Scraped Mazevo from %s to %s!", eventsStart.Format(time.DateTime), eventsEnd.Format(time.DateTime))
+
+		// Write event data to output file
+		fptr, err := os.Create(fmt.Sprintf("%s/%s.json", outDir, eventsStart.Format("2006-01")))
+		if err != nil {
+			log.Panic(err)
+		}
+		_, err = fptr.Write(bodyBytes)
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Click next month
+		err = chromedp.Click("[aria-label=\"Move to Next Month\"]", chromedp.NodeVisible).Do(ctx)
+		if err != nil {
+			log.Panic(err)
+		}
+		return nil
 	}
-	_, err = fptr.Write([]byte(stringBody))
+
+	_, err = chromedp.RunResponse(ctx,
+		chromedp.Navigate("https://east.mymazevo.com/calendar/4219c6df695c03860350ea213837fe59"),
+		chromedp.Sleep(5*time.Second),
+		chromedp.Click("input#displayMonth", chromedp.NodeVisible),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			for range 12 { // Scrape 12 months
+				err := scrapeLoop(ctx)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		}),
+	)
 	if err != nil {
-		panic(err)
+		log.Panic(err)
 	}
-	fptr.Close()
+
 }