Initial commit

.gitlab-ci.yml

+stages:
+  - build
+  - release
+
+variables:
+  GOLANG_VERSION: "1.22"  # Adjust to the Go version you are using
+
+build:
+  image: golang:${GOLANG_VERSION}
+  stage: build
+  services:
+    - docker
+  before_script:
+    - mkdir -p dist/
+  script:
+    - echo "Building for Windows"
+    - GOOS=windows GOARCH=amd64 go build -o dist/falcon_windows.exe
+    - echo "Building for macOS"
+    - GOOS=darwin GOARCH=amd64 go build -o dist/falcon_mac
+    - echo "Building for Linux"
+    - GOOS=linux GOARCH=amd64 go build -o dist/falcon_linux
+  artifacts:
+    paths:
+      - dist/
+
+release:
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  stage: release
+  script:
+    - echo "Creating a release"
+  release:
+    name: "Release v${CI_COMMIT_TAG}"
+    tag_name: "${CI_COMMIT_TAG}"
+    description: "Release of falcon version for all platforms."
+    assets:
+      links:
+        - name: "Download Windows version"
+          url: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/dist/falcon_windows.exe"
+        - name: "Download macOS version"
+          url: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/dist/falcon_mac"
+        - name: "Download Linux version"
+          url: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/dist/falcon_linux"
\ No newline at end of file

LICENSE

+# MIT License
+
+Copyright (c) 2024 Bayeriscer Rundfunk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

+# FALCON - Fetch and Log Continuous Observations of Networks
+
+FALCON is a tool for continuously fetching websites and logging performance metrics like page load time, element rendering, and more. It provides a flexible way to monitor websites and PNG file performance over time, with customizable sleep intervals and automatic logging of metrics.
+
+## Features
+
+- **Continuous Website and PNG Fetching**: Continuously fetches a list of websites or PNG files, logs the time it takes to load the entire page, and allows for the waiting of specific elements to fully render.
+- **Customizable Sleep Intervals**: Allows users to define a minimum and maximum sleep time between requests to prevent constant polling.
+- **Save and Load Configuration**: Automatically saves the user’s input (URLs, specific elements to wait for, sleep intervals) into a configuration file (`config.json`) and loads it on startup.
+- **Metrics Collection**: Logs the time for a full page load and specific elements to become visible, and provides detailed metrics like minimum, maximum, and average load times for each URL.
+- **CSV Logging**: Saves the performance data in a CSV file (`fetch_times.csv`), making it easy to analyze trends and performance over time.
+
+## How It Works
+
+### Fetch and Load Mechanism
+
+FALCON uses **Chromedp** to load websites and PNG files, and it operates by:
+
+1. **Fetching PNG Files**:
+    - For PNG files, the fetch mechanism simply requests the file and logs the time it took to fully receive it.
+
+2. **Fetching Websites**:
+    - For websites, FALCON waits for the entire page to load, including all static assets (such as images, CSS, JavaScript) by waiting for the `load` event.
+    - FALCON also waits for a **user-specified element** to become visible. This ensures that dynamic content (like JavaScript-rendered elements) is fully loaded before calculating the total time.
+    - FALCON adds a small buffer by waiting for **network idle** conditions to ensure all asynchronous loading is completed.
+
+### Collected Metrics
+
+1. **Full Page Load Time**:
+    - The time from the start of the navigation to the point where the entire page (including all images, stylesheets, and scripts) is loaded.
+   
+2. **Element Render Time**:
+    - The time it takes for a specific element (e.g., footer, main content) to become visible. This can be useful for monitoring dynamically loaded content.
+
+3. **Sleep Between Fetches**:
+    - A user-specified **min** and **max** sleep interval is used to control how often FALCON fetches the URLs, ensuring it doesn’t overload the server with constant requests.
+
+4. **Minimum, Maximum, and Average Load Time**:
+    - FALCON calculates the minimum, maximum, and average load time for each URL, providing insight into the performance over multiple fetch attempts.
+
+## Installation
+
+### Prerequisites
+
+- **Go** (version 1.19 or higher)
+- **Chromedp**: FALCON uses Chromedp, a Go library for driving headless Chrome.
+
+You can install Chromedp by running:
+
+```bash
+go get -u github.com/chromedp/chromedp
+```
+
+### Build and Run
+
+To build and run FALCON from source:
+
+```bash
+git clone https://gitlab.com/<your-username>/<your-repo>.git
+cd <your-repo>
+go build -o falcon
+./falcon
+```
+
+For specific platforms:
+
+- **Windows**: `GOOS=windows GOARCH=amd64 go build -o falcon.exe`
+- **Linux**: `GOOS=linux GOARCH=amd64 go build -o falcon`
+- **Mac**: `GOOS=darwin GOARCH=amd64 go build -o falcon`
+
+## Usage
+
+1. Enter URLs: Provide a list of URLs (one per line) in the URL input field.
+2. Enter CSS Selector: Specify a CSS selector (e.g., #footer, .content) that should be visible before calculating the load time.
+3. Set Sleep Intervals: Define the minimum and maximum sleep intervals (in seconds) between fetch attempts.
+4. Start: Click the Start button to begin fetching the URLs continuously.
+5. Stop: Click the Stop button to halt the fetching process.
+
+## Configuration File
+
+FALCON automatically saves your input into a configuration file (config.json) when you start fetching. On subsequent runs, this configuration will be loaded automatically, pre-filling the input fields.
+
+The config.json file has the following structure:
+
+```json
+{
+  "urls": ["https://example.com", "https://another.com"],
+  "element": "#footer",
+  "min_interval": 5,
+  "max_interval": 10
+}
+```
+
+You can manually edit this file to change the URLs, element, or intervals.
+
+## Metrics Example
+
+The following metrics are collected for each URL:
+
+- Min Load Time: The shortest time taken to load the full page.
+- Max Load Time: The longest time taken to load the full page.
+- Average Load Time: The average time across all fetches.
+
+The CSV file (fetch_times.csv) will contain logs like this:
+
+```csv
+Timestamp,URL,Load,TTFB,LCP
+2024-10-05T12:34:56Z,https://example.com,2.34s,1.23s,1.45s
+2024-10-05T12:36:00Z,https://example.com,2.11s,1.12s,1.34s
+2024-10-05T12:37:30Z,https://another.com,1.45s,0.89s,1.12s
+```
+
+## Download
+
+You can download the latest release for your platform below:
+
+- [Windows version](https://gitlab.ard.de/apps/falcon/-/releases/permalink/latest/download/falcon_windows.exe)
+- [macOS version](https://gitlab.ard.de/apps/falcon/-/releases/permalink/latest/download/falcon_mac)
+- [Linux version](https://gitlab.ard.de/apps/falcon/-/releases/permalink/latest/download/falcon_linux)
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

config.json

+{
+  "urls": [
+    "https://ardzsd.de/otrs/index.pl?Action=AgentTicketPhone",
+    "https://ardzsd.de/otrs-web/skins/Agent/BR/img/ARD-ServiceDesk_RGB_72dpi.png"
+  ],
+  "element": "#submitRichText",
+  "min_interval": 10,
+  "max_interval": 120
+}
\ No newline at end of file

go.mod

+module gitlab.ard.de/apps/falcon
+
+go 1.22.5
+
+require (
+	fyne.io/fyne/v2 v2.5.1
+	github.com/chromedp/chromedp v0.10.0
+)
+
+require (
+	fyne.io/systray v1.11.0 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/chromedp/cdproto v0.0.0-20241003230502-a4a8f7c660df // indirect
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fredbi/uri v1.1.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fyne-io/gl-js v0.0.0-20220119005834-d2da28d9ccfe // indirect
+	github.com/fyne-io/glfw-js v0.0.0-20240101223322-6e1efdc71b7a // indirect
+	github.com/fyne-io/image v0.0.0-20220602074514-4956b0afb3d2 // indirect
+	github.com/go-gl/gl v0.0.0-20211210172815-726fda9656d6 // indirect
+	github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect
+	github.com/go-text/render v0.1.1-0.20240418202334-dd62631dae9b // indirect
+	github.com/go-text/typesetting v0.1.0 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.4.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gopherjs/gopherjs v1.17.2 // indirect
+	github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rymdport/portal v0.2.6 // indirect
+	github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect
+	github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/yuin/goldmark v1.7.1 // indirect
+	golang.org/x/image v0.18.0 // indirect
+	golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

main.go

+package main
+
+import (
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"fyne.io/fyne/v2/app"
+	"fyne.io/fyne/v2/container"
+	"fyne.io/fyne/v2/widget"
+	"github.com/chromedp/chromedp"
+	"github.com/chromedp/cdproto/performance"
+)
+
+// Config struct for saving and loading user inputs
+type Config struct {
+	URLs        []string `json:"urls"`
+	Element     string   `json:"element"`
+	MinInterval int      `json:"min_interval"`
+	MaxInterval int      `json:"max_interval"`
+}
+
+var configFile = "config.json"
+var stopFetching bool
+var running bool
+
+// Fetches a PNG file or website and logs the time
+func fetchImage(url string) (time.Duration, error) {
+	start := time.Now()
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("failed to fetch: %s", resp.Status)
+	}
+
+	duration := time.Since(start)
+	return duration, nil
+}
+
+// Fetch website metrics including wait for load, element visibility, LCP, TTFB, and network idle
+func fetchWebsiteMetrics(url, element string) (total time.Duration, lcp time.Duration, ttfb time.Duration, err error) {
+	start := time.Now()
+
+	// Create a new Chromedp context
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	var metrics []*performance.Metric
+
+	// Run Chromedp to navigate to the URL, wait for the load event, wait for element visibility, and ensure network idle
+	err = chromedp.Run(ctx,
+		chromedp.Navigate(url),                // Navigate to the website
+		chromedp.WaitVisible(element),         // Wait for a specific user-defined element to be visible
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			var err error
+			metrics, err = performance.GetMetrics().Do(ctx)
+			return err
+		}),
+		chromedp.Sleep(2*time.Second),         // Ensure network idle by waiting a few seconds after
+	)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	// Calculate total load time after waiting for all conditions
+	total = time.Since(start)
+
+	// Extract LCP and TTFB from performance metrics
+	lcp = performanceMetric(metrics, "LargestContentfulPaint")
+	ttfb = performanceMetric(metrics, "NavigationTiming.responseStart") - performanceMetric(metrics, "NavigationTiming.requestStart")
+
+	return total, lcp, ttfb, nil
+}
+
+// Helper function to get specific metric from the performance metrics
+func performanceMetric(metrics []*performance.Metric, name string) time.Duration {
+	for _, metric := range metrics {
+		if metric.Name == name {
+			return time.Duration(metric.Value) * time.Millisecond
+		}
+	}
+	return 0
+}
+
+// Logs the fetch details to CSV including total load time, LCP, and TTFB
+func logTimeToCSV(url string, totalDuration, lcp, ttfb time.Duration, filename string) error {
+	file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+
+	// Log the current timestamp, URL, total load time, LCP, and TTFB
+	record := []string{
+		time.Now().Format(time.RFC3339),
+		url,
+		totalDuration.String(), // Total load time
+		lcp.String(),           // Largest Contentful Paint (LCP)
+		ttfb.String(),          // Time to First Byte (TTFB)
+	}
+	return writer.Write(record)
+}
+
+// Calculate and display metrics: min, max, and average load times
+func calculateAndDisplayMetrics(url string, durations []time.Duration, logLabel *widget.Label) {
+	if len(durations) == 0 {
+		logLabel.SetText(fmt.Sprintf("No data for %s", url))
+		return
+	}
+
+	var total time.Duration
+	min, max := durations[0], durations[0]
+
+	for _, d := range durations {
+		total += d
+		if d < min {
+			min = d
+		}
+		if d > max {
+			max = d
+		}
+	}
+
+	average := total / time.Duration(len(durations))
+
+	logLabel.SetText(fmt.Sprintf("URL: %s\nMin Load Time: %v\nMax Load Time: %v\nAvg Load Time: %v", url, min, max, average))
+}
+
+// Save the user inputs to a JSON config file
+func saveConfig(urls []string, element string, minInterval, maxInterval int) error {
+	config := Config{
+		URLs:        urls,
+		Element:     element,
+		MinInterval: minInterval,
+		MaxInterval: maxInterval,
+	}
+
+	file, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(configFile, file, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Load configuration from the config file if it exists
+func loadConfig() (*Config, error) {
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	file, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var config Config
+	err = json.Unmarshal(file, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+func main() {
+	a := app.New()
+	w := a.NewWindow("FALCON - Fetch Time Logger with Config")
+
+	// Multi-line input field for URLs
+	urlListEntry := widget.NewMultiLineEntry()
+	urlListEntry.SetPlaceHolder("Enter URLs, one per line")
+
+	// Input field for specific element to wait for
+	elementEntry := widget.NewEntry()
+	elementEntry.SetPlaceHolder("Enter CSS selector of element to wait for (e.g., #footer)")
+
+	// Input fields for custom sleep intervals
+	minInterval := widget.NewEntry()
+	minInterval.SetPlaceHolder("Min seconds")
+
+	maxInterval := widget.NewEntry()
+	maxInterval.SetPlaceHolder("Max seconds")
+
+	// Load configuration if available and pre-fill inputs
+	config, err := loadConfig()
+	if err == nil && config != nil {
+		urlListEntry.SetText(strings.Join(config.URLs, "\n"))
+		elementEntry.SetText(config.Element)
+		minInterval.SetText(strconv.Itoa(config.MinInterval))
+		maxInterval.SetText(strconv.Itoa(config.MaxInterval))
+	}
+
+	// Log display
+	logLabel := widget.NewLabel("Ready")
+
+	// Buttons for start and stop fetching
+	startButton := widget.NewButton("Start", func() {
+		if running {
+			logLabel.SetText("Already running...")
+			return
+		}
+		stopFetching = false
+		running = true
+
+		go func() {
+			// Parse the list of URLs
+			urls := strings.Split(urlListEntry.Text, "\n")
+			urls = cleanURLList(urls)
+
+			// Get the specific element to wait for
+			elementToWaitFor := elementEntry.Text
+			if elementToWaitFor == "" {
+				logLabel.SetText("Please enter a valid CSS selector for the element to wait for.")
+				running = false
+				return
+			}
+
+			// Parse sleep interval range
+			min, errMin := strconv.Atoi(minInterval.Text)
+			max, errMax := strconv.Atoi(maxInterval.Text)
+			if errMin != nil || errMax != nil || min <= 0 || max <= min {
+				logLabel.SetText("Invalid sleep intervals. Please provide valid min/max seconds.")
+				running = false
+				return
+			}
+
+			// Save the configuration
+			err := saveConfig(urls, elementToWaitFor, min, max)
+			if err != nil {
+				logLabel.SetText(fmt.Sprintf("Error saving config: %v", err))
+				running = false
+				return
+			}
+
+			// Store load times for each URL
+			urlMetrics := make(map[string][]time.Duration)
+
+			for !stopFetching {
+				for _, url := range urls {
+					if stopFetching {
+						logLabel.SetText("Stopped fetching.")
+						break
+					}
+
+					var duration time.Duration
+					var lcp, ttfb time.Duration
+					var err error
+
+					// Determine if it's a PNG or website
+					if len(url) > 4 && url[len(url)-4:] == ".png" {
+						// Fetch PNG
+						duration, err = fetchImage(url)
+					} else {
+						// Fetch website with advanced loading metrics
+						duration, lcp, ttfb, err = fetchWebsiteMetrics(url, elementToWaitFor)
+					}
+
+					if err != nil {
+						logLabel.SetText(fmt.Sprintf("Error fetching %s: %v", url, err))
+						break
+					}
+
+					// Log the fetch time
+					err = logTimeToCSV(url, duration, lcp, ttfb,  "fetch_times.csv")
+					if err != nil {
+						logLabel.SetText(fmt.Sprintf("Error logging time: %v", err))
+						break
+					}
+
+					// Add the duration to the metrics for this URL
+					urlMetrics[url] = append(urlMetrics[url], duration)
+
+					// Calculate and display metrics for this URL
+					calculateAndDisplayMetrics(url, urlMetrics[url], logLabel)
+
+					// Sleep between requests
+					sleepDuration := min + rand.Intn(max-min+1)
+					logLabel.SetText(fmt.Sprintf("Fetched %s in %v, sleeping for %d seconds", url, duration, sleepDuration))
+					time.Sleep(time.Duration(sleepDuration) * time.Second)
+				}
+			}
+
+			running = false
+			logLabel.SetText("All URLs processed.")
+		}()
+	})
+
+	stopButton := widget.NewButton("Stop", func() {
+		if running {
+			stopFetching = true
+			running = false
+			logLabel.SetText("Stopped fetching.")
+		} else {
+			logLabel.SetText("Not running.")
+		}
+	})
+
+	// Layout: URL List, Element Input, Intervals, Log Label, Start/Stop Buttons
+	content := container.NewVBox(
+		urlListEntry,
+		elementEntry,
+		widget.NewLabel("Min Interval (seconds)"),
+		minInterval,
+		widget.NewLabel("Max Interval (seconds)"),
+		maxInterval,
+		logLabel,
+		container.NewHBox(startButton, stopButton),
+	)
+
+	w.SetContent(content)
+	w.ShowAndRun()
+}
+
+// Helper function to clean the URL list (remove empty lines and trim spaces)
+func cleanURLList(urls []string) []string {
+	var cleanedURLs []string
+	for _, url := range urls {
+		trimmed := strings.TrimSpace(url)
+		if trimmed != "" {
+			cleanedURLs = append(cleanedURLs, trimmed)
+		}
+	}
+	return cleanedURLs
+}
\ No newline at end of file