RELIVAL/CODE/regenerate_data.R

# This code cannot be reproduced without access to the underlying datasets
data_sets = c('aclew', 'lena')
corpora <- c('bergelson', 'lucid', 'winnipeg', 'warlaumont','cougar','fausey-trio','lyon','quechua') 
no.scale.columns <- c('experiment', 'session_id', 'child_id','child_id_unique','age_s',
                      'date_iso', 'child_dob', 'missing_audio',"age_bin","duration","usession_id",
                      "normative","age","duration_alice", "duration_vcm" ,  "duration_vtc","duration_its" )


for (data_set in data_sets){ #data_set="aclew"
  mydat <- read.csv(paste0('../input/el1000-metrics/output/', data_set,'_metrics.csv'))
  mydat <- mydat[is.element(mydat$experiment, corpora),]
  
  mydat2 <- read.csv(paste0('../input/laac-metrics/output/', data_set,'_metrics.csv'))
  mydat2 <- mydat2[mydat2$experiment %in% corpora,]
  
  # Remove FauseyElse
  fausey_trio <- read.csv('../input/laac-metrics/datasets/fausey-trio/metadata/recordings.csv')
  fausey_trio_full <- fausey_trio[fausey_trio$Trio_Subset == "Trio_Full", ]
  fausey_trio_full$session_id <- paste0(fausey_trio_full$HomeBank_ID, "/", fausey_trio_full$fileName)
  mydat2 <- mydat2[mydat2$session_id %in% fausey_trio_full$session_id,]
  #note that since we are only taking fausey-trio, then the fact that png2019 & tsimane2017 are repeated across laac & el1000 is not a problem
  
  mydat=rbind(mydat,mydat2)
  
  # Remove Cougar non-normatives
  cougar <- read.csv('../input/el1000-metrics/EL1000/cougar/metadata/children.csv')
  cougar_normative <- cougar[cougar$normative == "Y", ]
  mydat <- mydat[mydat$experiment != 'cougar' | mydat$child_id %in% cougar_normative$child_id, ]
  
  # Save data
  write.csv(mydat,paste0('../data_output/', data_set,'_base_data_set.csv'),row.names = F)
  print(paste0('Save to ', paste0('../data_output/', data_set,'_base_data_set.csv')))
}


for (data_set in data_sets){
  mydat <- read.csv(paste0('../data_output/', data_set,'_base_data_set.csv'))
  mydat$age_s=scale(mydat$age)
  mydat$age_s=(mydat$age - mean(mydat$age , na.rm=T))/sd(mydat$age , na.rm=T)

  
  #remove outliers
  for(metric in metrics) mydat[abs((mydat[,metric]-mean(mydat[,metric], na.rm=T))/sd(mydat[,metric], na.rm=T)) > 2.5 , metric]<-NA 
  #NA values that are beyond 2.5 SD from mean
  
    
  write.csv(mydat,paste0('../data_output/', data_set,'_metrics.csv'),row.names = F) #all variables are unscaled, except for age
  print(paste0('Save to ', paste0('../data_output/', data_set,'_metrics.csv')))
  
  metrics = colnames(mydat)[!is.element(colnames(mydat), no.scale.columns)]
  for(metric in metrics){ #metric="pc_mal_ph"
   
    
    # Scale mydat
    mydat[, metric] <- (mydat[, metric] - mean(mydat[, metric], na.rm=T))/sd(mydat[, metric], na.rm=T)
  }
  
  # Save data
  write.csv(mydat,paste0('../data_output/', data_set,'_metrics_scaled.csv'),row.names = F)
  print(paste0('Save to ', paste0('../data_output/', data_set,'_metrics_scaled.csv')))
}