Sys.setenv(TZ = "UTC") 
#library(tidyverse)
library(lubridate)

###################################################################
# SELECT TIME PERIOD
# seconds for  2 days
daystep <- 60*60*24
step <- daystep*7
args <- commandArgs(trailingOnly = TRUE)
message("ARGUMENTS = ",length(args))
if (length(args)==0) {
  t2 <- as.POSIXlt(Sys.time(),tz="UTC")
  #t2 <- Sys.time()
  t1 <- t2 - step
  #"args:" length(args)
} else {
  t1 <- as.POSIXct(args[1])
  t2 <- as.POSIXct(args[2])
}

# SET TIME PERIOD HERE:
#t1 <- as.POSIXct("2021-11-25 00:00:00 CET")
#t2 <- as.POSIXct("2021-11-25 23:59:59 CET")
t1 <- as.POSIXct("2023-05-23 14:00:00 UTC")
#t2 <- as.POSIXlt(Sys.time(),tz="UTC")
t2 <- as.POSIXct("2023-08-22 06:00:00 UTC")
t2 <- round_date(as.POSIXct(t2),"5 minutes")

t1 <- as.POSIXlt(format(t1,"%Y-%m-%d %H:%M:00"))
t2 <- as.POSIXlt(format(t2,"%Y-%m-%d %H:%M:00"))
#t2 <- t2 + 60*60



message("PERIOD = ",t1," ",t2)
runningdate <- seq.Date( from=as.Date(t1), to=as.Date(t2), by="day")

aggregation <- "60 sec"
aggregation <- "1 min"
aggregation <- "5 min"
#aggregation <- "15 min"



###################################################################
# GET MATCHING FILE NAMES FOR SELECTED DAYS
flist <- NULL
for (i in seq_along(runningdate)) {
  name1 <- runningdate[i]
  name2 <- "OPC-004"
  name3 <- "txt"
  url <- paste("https://saqn.geo.uni-augsburg.de/grimm.cgi?name1=",name1,"&name2=",name2,"&name3=",name3,sep="")
  message("URL = ",url)
  flist_tmp <- try(read.table(url,stringsAsFactors=F))
  if(inherits(flist_tmp, "try-error")){
    message("no files")
  }else{
    #message("FILE LIST = ",flist_tmp)
    flist <- rbind(flist,flist_tmp)
  }
}

###################################################################
# READ MATCHING FILES DATA IN OBJECT asopc
opcheader <- c("datetime","serialno","type","PM10","PM25","PM1","um035","um046","um066","um100","um130",
            "um170","um230","um300","um400","um520","um650","um800","um1000","um1200","um1400","um1600",
            "um1800","um2000","um2200","um2500","um2800","um3100","um3400","um3700","serialOPC","Tout",
            "Taerosol","rH","UTC","longitude","latitude","height","timeresolution","softwareversion","lastcalibration","configversion")

edmheader <- c("datetime","UTC","longitude","latitude","height","PM10","PM4","PM25","PM1","um0_25","um0_28","um0_30",
            "um0_35","um0_40","um0_45","um0_50","um0_58","um0_65","um0_70","um0_80","um1_00","um1_30","um1_60","um2_00",
            "um2_50","um3_00","um3_50","um4_00","um5_00","um6_50","um7_50","um8_50","um10_00","um12_50","um15_00","um17_50",
            "um20_00","um25_00","um30_00","um32_00","Tout","rH","pressure","serialno","softwareversion")

#asopc <- data.frame(
#  datetime=character(),serialno=character(),type=character(),PM10=double(),PM25=double(),PM1=double(),um035=double(),um046=double(),
#  um066=double(),um100=double(),um130=double(),um170=double(),um230=double(),um300=double(),um400=double(),um520=double(),
#  um650=double(),um800=double(),um1000=double(),um1200=double(),um1400=double(),um1600=double(),um1800=double(),um2000=double(),
#  um2200=double(),um2500=double(),um2800=double(),um3100=double(),um3400=double(),um3700=double(),serialOPC=character(),
#  Tout=double(),Taerosol=double(),rH=double(),UTC=character(),longitude=double(),latitude=double(),height=double(),
#  timeresolution=character(),softwareversion=character(),lastcalibration=character(),configversion=character(),
#  stringsAsFactors=FALSE)

asopc <- data.frame(
  opcid=character(),datetime=character(),serialno=character(),PM10=double(),PM25=double(),PM1=double(),
  Tout=double(),rH=double(),UTC=character(),longitude=double(),latitude=double(),height=double(),
  stringsAsFactors=FALSE)

opcid <- ""
for (i in seq_along(flist$V1)) {
  opcid <- strsplit(flist$V1[i],"/",fixed=T)[[1]][2]

  fname <- paste("https://saqn.geo.uni-augsburg.de/grimm/",flist$V1[i],sep="")
  message(opcid," FILE ADRESS = ",fname)

  opcidtype <- strsplit(opcid,"-",fixed=T)[[1]][1] # "OPC"

   if(opcid=="OPC-002") opcid <- "OPC002"
   if(opcid=="OPC-004") opcid <- "OPC004"
   if(opcid=="OPC-006") opcid <- "OPC006"


###########################
# append empty line for OPC-004:
# df[nrow(df)+1,] <- NA

  if (opcidtype == "OPC") {
    x <- try(read.table(fname,header=F,stringsAsFactors=F,comment.char="*",sep=";",dec=",",skip=0,col.names=edmheader,fill=T))
  } else {
    x <- try(read.table(fname,header=F,stringsAsFactors=F,comment.char="*",sep=";",dec=",",skip=0,col.names=opcheader,fill=T))
  }

    #message(str(asopc))
    #message(typeof(asopc))
    if( typeof(x) == "character"){
      next
    }
    if( is.data.frame(x) == F ){
      next
    }
    #message("nrow = ",nrow(asopc))
    if (nrow(x)<1) {
       next
    }
    #asopc$opctype <- opctype
    x$opcid <- opcid
    #asopc$shttype <- shttype
    #asopc$shtid <- shtid
    #asopc[nrow(asopc)+1,] <- NA
    #asopc <- merge(asopc,asopc_tmp,all.x =TRUE)
    
    asopc_tmp <- data.frame(opcid=x$opcid, datetime=x$datetime, serialno=x$serialno, PM10=x$PM10, PM25=x$PM25, PM1=x$PM1, 
                            Tout=x$Tout, rH=x$rH, UTC=x$UTC, longitude=x$longitude, latitude=x$latitude, height=x$height)

    asopc <- rbind(asopc,asopc_tmp)

}
#asopc$time <- as.POSIXct(asopc$UTC,format="%Y-%m-%d %H:%M:%S")
#asopc$time <- as.POSIXct(asopc$datetime,format="%Y-%m-%d %H:%M:%S") - 60*60

#asopc$time <- as.POSIXct(asopc$UTC,format="%Y-%m-%d %H:%M:%S") - 30*60
asopc$time <- as.POSIXlt(asopc$UTC,format="%Y-%m-%d %H:%M:%S",tz="UTC") 

asopc[as.character(asopc)=="nan"] <- NA

asopc$PM10 <- as.numeric(asopc$PM10)
asopc$PM25 <- as.numeric(asopc$PM25)
asopc$PM1 <- as.numeric(asopc$PM1)
asopc$Tout <- as.numeric(asopc$Tout)
asopc$rH <- as.numeric(asopc$rH)
asopc$longitude <- as.numeric(asopc$longitude)
asopc$latitude <- as.numeric(asopc$latitude)
asopc$height <- as.numeric(asopc$height)

message("reading done!")


###################################################################
# CREATE DATA FRAME OF aggregation INTERVALL TIME STEPS
#secs <- seq.POSIXt(from=as.POSIXct(t1),to=as.POSIXct(t2),by="sec")
#secs <- data.frame( time=seq.POSIXt(from=as.POSIXct(t1),to=as.POSIXct(t2),by=aggregation))
#secs <- data.frame( time=seq(from=as.POSIXct(t1),to=as.POSIXct(t2),by=aggregation))
timesteps <- data.frame( time=seq(from=as.POSIXct(t1),to=as.POSIXct(t2),by=aggregation))
aggpm10 <- timesteps
aggtemp <- timesteps
aggrhum <- timesteps
aggpm25 <- timesteps


ids <- sort(unique(asopc$opcid))
for (i in seq_along(ids)) {

   message(" ")
   message("id =",ids[i])

#   idserpm10 <- c(0.0,asopc$PM10[asopc$opcid==ids[i]])
#   idserpm25 <- c(0.0,asopc$PM25[asopc$opcid==ids[i]])
#   idsertemp <- c(0.0,asopc$Tout[asopc$opcid==ids[i]])
#   idserrhum <- c(0.0,asopc$rH[asopc$opcid==ids[i]])
   idserpm10 <- asopc$PM10[asopc$opcid==ids[i]]
   idserpm25 <- asopc$PM25[asopc$opcid==ids[i]]
   idsertemp <- asopc$Tout[asopc$opcid==ids[i]]
   idserrhum <- asopc$rH[asopc$opcid==ids[i]]

   #idser[1] <- NA
   #idtime <- c(as.POSIXct(t1),asopc$time[asopc$opcid==ids[i]])
   #idtime <- c(as.POSIXct(runningdate[1]),asopc$time[asopc$opcid==ids[i]])
   idtime <- asopc$time[asopc$opcid==ids[i]]
   tmppm10 <- data.frame(time=idtime,var=idserpm10)
   tmppm25 <- data.frame(time=idtime,var=idserpm25)
   tmptemp <- data.frame(time=idtime,var=idsertemp)
   tmprhum <- data.frame(time=idtime,var=idserrhum)

   # FILTER
   tmppm10$var[tmppm10$var > 60 ] <- NA


   #tmp$cuts <- cut( idtime,breaks=aggregation)
   #cuts <- cut( idtime,breaks=timesteps$time)
   secstime1 <- as.numeric(timesteps$time)
   idtime1 <- as.numeric(idtime)
   cuts <- cut( idtime1,breaks=secstime1,labels=format(timesteps$time[1:length(secstime1)-1],"%Y-%m-%d %H:%M:%S UTC"))

   if ( all(is.na(cuts)) ) {
     aggpm10$var <- NA
     names(aggpm10)[length(names(aggpm10))] <- paste0("id",ids[i])
     aggpm25$var <- NA
     names(aggpm25)[length(names(aggpm25))] <- paste0("id",ids[i])
     aggtemp$var <- reg$var
     names(aggtemp)[length(names(aggtemp))] <- paste0("id",ids[i])
     aggrhum$var <- reg$var
     names(aggrhum)[length(names(aggrhum))] <- paste0("id",ids[i])
     next
   }
   tmppm10$cuts <- cuts
   tmppm25$cuts <- cuts
   tmptemp$cuts <- cuts
   tmprhum$cuts <- cuts

   message("pm10:",str(tmppm10))
   if( all(is.na(tmppm10$var)) ) {
    aggpm10$pm10 <- as.numeric(NA)
    names(aggpm10)[length(names(aggpm10))] <- paste0("id",ids[i])
   } else {
   agg <- aggregate(. ~ cuts, tmppm10, mean)
   agg$time <- as.POSIXct(agg$cuts)
   reg <- merge(timesteps,agg,by="time",all.x=T)
   aggpm10$var <- reg$var
   names(aggpm10)[length(names(aggpm10))] <- paste0("id",ids[i])
   }

   message("pm25:",str(tmppm25))
   if( all(is.na(tmppm25$var)) ) {
    aggpm25$pm10 <- as.numeric(NA)
    names(aggpm25)[length(names(aggpm25))] <- paste0("id",ids[i])
   } else {
   agg <- aggregate(. ~ cuts, tmppm25, mean)
   agg$time <- as.POSIXct(agg$cuts)
   reg <- merge(timesteps,agg,by="time",all.x=T)
   aggpm25$var <- reg$var
   names(aggpm25)[length(names(aggpm25))] <- paste0("id",ids[i])
   }

   message("temp:",str(tmptemp))
   if( all(is.na(tmptemp$var)) ) {
    aggtemp$temp <- as.numeric(NA)
    names(aggtemp)[length(names(aggtemp))] <- paste0("id",ids[i])
   } else {
   agg <- aggregate(. ~ cuts, tmptemp, mean)
   agg$time <- as.POSIXct(agg$cuts)
   reg <- merge(timesteps,agg,by="time",all.x=T)
   aggtemp$var <- reg$var
   names(aggtemp)[length(names(aggtemp))] <- paste0("id",ids[i])
   }

   message("rhum:",str(tmprhum))
   if( all(is.na(tmprhum$var)) ) {
    aggrhum$rhum <- as.numeric(NA)
    names(aggrhum)[length(names(aggrhum))] <- paste0("id",ids[i])
   } else {
   agg <- aggregate(. ~ cuts, tmprhum, mean)
   agg$time <- as.POSIXct(agg$cuts)
   reg <- merge(timesteps,agg,by="time",all.x=T)
   aggrhum$var <- reg$var
   names(aggrhum)[length(names(aggrhum))] <- paste0("id",ids[i])
   }

}
message("aggregation done!")


aggpm10 <- setNames(aggpm10, replace(names(aggpm10), names(aggpm10) == 'idOPC-002', 'idOPC002'))
aggpm10 <- setNames(aggpm10, replace(names(aggpm10), names(aggpm10) == 'idOPC-004', 'idOPC004'))
aggpm10 <- setNames(aggpm10, replace(names(aggpm10), names(aggpm10) == 'idOPC-006', 'idOPC006'))

aggpm25 <- setNames(aggpm25, replace(names(aggpm25), names(aggpm25) == 'idOPC-002', 'idOPC002'))
aggpm25 <- setNames(aggpm25, replace(names(aggpm25), names(aggpm25) == 'idOPC-004', 'idOPC004'))
aggpm25 <- setNames(aggpm25, replace(names(aggpm25), names(aggpm25) == 'idOPC-006', 'idOPC006'))

aggtemp <- setNames(aggtemp, replace(names(aggtemp), names(aggtemp) == 'idOPC-002', 'idOPC002'))
aggtemp <- setNames(aggtemp, replace(names(aggtemp), names(aggtemp) == 'idOPC-004', 'idOPC004'))
aggtemp <- setNames(aggtemp, replace(names(aggtemp), names(aggtemp) == 'idOPC-006', 'idOPC006'))

aggrhum <- setNames(aggrhum, replace(names(aggrhum), names(aggrhum) == 'idOPC-002', 'idOPC002'))
aggrhum <- setNames(aggrhum, replace(names(aggrhum), names(aggrhum) == 'idOPC-004', 'idOPC004'))
aggrhum <- setNames(aggrhum, replace(names(aggrhum), names(aggrhum) == 'idOPC-006', 'idOPC006'))

edm <- asopc
edmaggpm10 <- aggpm10
edmaggpm25 <- aggpm25
edmaggtemp <- aggtemp
edmaggrhum <- aggrhum

save(edm,edmaggpm10,edmaggpm25,edmaggrhum,edmaggtemp,file="edm.Rdata")

message("writing grimm.Rdata done!")