package main import ( "bufio" "compress/gzip" "encoding/json" "flag" "fmt" "io" "log" "os" "path/filepath" "regexp" "strings" ) type nginxLog struct { Address string `json:"address"` Host string `json:"host"` Local_time string `json:"local_time"` Method string `json:"method"` Msec_time float64 `json:"msec_time"` Request_length int `json:"request_length"` Resp_body_size int `json:"resp_body_size"` Resp_time float64 `json:"resp_time"` Status int `json:"status"` Upstream_addr string `json:"upstream_addr"` Uri string `json:"uri"` User_agent string `json:"user_agent"` } func readFile(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", err } defer file.Close() var fileContent strings.Builder scanner := bufio.NewScanner(file) for scanner.Scan() { fileContent.WriteString(scanner.Text()) fileContent.WriteString("\n") } return fileContent.String(), scanner.Err() } func readLogLines(logs *[]nginxLog, data string) { var log nginxLog lines := strings.Split(data, "\n") for _, line := range lines { if line == "" { continue } err := json.Unmarshal([]byte(line), &log) if err != nil { fmt.Printf("Error unmarshalling JSON: %v, line: %s\n", err, line) continue // Skip to the next line in case of error } *logs = append(*logs, log) } } func parseLogs(logGlob string) []nginxLog { files, err := filepath.Glob(logGlob) gzipFile := regexp.MustCompile(`\.gz$`) if err != nil { log.Fatal(err) } var logs []nginxLog for count, fileName := range files { if gzipFile.MatchString(fileName) { file, err := os.Open(fileName) if err != nil { panic(err) } defer file.Close() // Create a gzip reader gzipReader, err := gzip.NewReader(file) if err != nil { panic(err) } defer gzipReader.Close() // Read the uncompressed contents content, err := io.ReadAll(gzipReader) if err != nil { panic(err) } fmt.Println(count, fileName, "gzip file") readLogLines(&logs, string(content)) } else { contents, err := readFile(fileName) if err != nil { log.Fatal(err) } readLogLines(&logs, contents) fmt.Println(count, fileName, "plain text") } } return logs } func main() { logGlob := flag.String("logs", "/var/log/nginx/*", "Log file(s) glob to parse") flag.Parse() logs := parseLogs(*logGlob) for _, log := range logs { fmt.Println(log) } }