Analysis2.Rmd

---
title: "Analysis 2- MIddle School transition with no transition"
author: "Arun Sharma"
date: "1/1/2020"
output: 
  html_document:
    theme: paper
    highlight: tango
    toc: true
    toc_depth: 5
---

```{r global_options, include=FALSE, fig.width = 10}
knitr::opts_chunk$set(warning=FALSE, message=FALSE)

```

```{r}
library(tidyverse)
library(ggplot2)
library(sqldf)
```

# Estimatng the growth pattern amongst students into the middle school


## Imports
```{r}
rm(list=ls())
library(data.table)

#getting the data for the year 2017-18
Fall1718 <- fread("2018_NWEA/NWEA Fall 2017-hashed.csv")
Winter1718 <- fread("2018_NWEA/NWEA_Winter_2017-2018-hashed.csv")
Spring1718 <- fread("2018_NWEA/NWEA_Spring_2017-2018-hashed.csv")


#getting the data for the year 2018-19
Fall1819 <- fread("2019_NWEA/NWEA Fall 2018-hashed.csv")
Winter1819 <- fread("2019_NWEA/NWEA_Winter_2018-2019-hashed.csv")
Spring1819 <- fread("2019_NWEA/NWEA_Spring_2018-2019-hashed.csv")

#read only elementary students from 17-18 and only middle school students from 18-19
combined.elementary1718 <- fread("2017-18 elementary.csv")
combined.middle1819 <- fread("2018-19 middleschool.csv")
```

```{r}
#identifying the middle school students , create a new column and set it to zero
combined.elementary1718$is.middle <- 0
combined.middle1819$is.middle <- 0

```


```{r}
LongData_1718 <- rbind(Fall1718,Winter1718,Spring1718)
LongData_1819 <- rbind(Fall1819,Winter1819,Spring1819)

student.analysis1 <- rbind(LongData_1718, LongData_1819)

#total number of students prior to analysis
total.number <- student.analysis1 %>%
  group_by(StudentID)%>%
  summarise(n())

nrow(total.number)
```
## Number of students in each schools for 17-18 and 18-19
```{r}
#Greenock
greenock1718 <- LongData_1718[LongData_1718$SchoolName == "Greenock Elementary"]
greenock1819 <- LongData_1819[LongData_1819$SchoolName == "Greenock Elementary"]

#central Elementary
central1718 <- LongData_1718[LongData_1718$SchoolName == "Central Elementary"]
central1819 <- LongData_1819[LongData_1819$SchoolName == "Central Elementary"]

#Mount Vernon Elementary
mountvernon1718 <- LongData_1718[LongData_1718$SchoolName == "Mount Vernon Elementary"]
mountvernon1819 <- LongData_1819[LongData_1819$SchoolName == "Mount Vernon Elementary"]

#William Penn Elementary
williampenn1718 <- LongData_1718[LongData_1718$SchoolName == "William Penn Elementary"]
williampenn1819 <- LongData_1819[LongData_1819$SchoolName == "William Penn Elementary"]

#Elizabeth Forward Middle School
Elizabethmiddle1718 <- LongData_1718[LongData_1718$SchoolName == "Elizabeth Forward Middle School"]
Elizabethmiddle1819 <- LongData_1819[LongData_1819$SchoolName == "Elizabeth Forward Middle School"]
```

```{r}
#preparing a table for students in each school
number.students <- c(rapply(greenock1718[,2], function(x)length(unique(x))),
rapply(greenock1819[,2], function(x)length(unique(x))),

rapply(central1718[,2], function(x)length(unique(x))),
rapply(central1819[,2], function(x)length(unique(x))),


rapply(williampenn1718[,2], function(x)length(unique(x))),
rapply(williampenn1819[,2], function(x)length(unique(x))),

rapply(mountvernon1718[,2], function(x)length(unique(x))),
rapply(mountvernon1819[,2], function(x)length(unique(x))),

rapply(Elizabethmiddle1718[,2], function(x)length(unique(x))),
rapply(Elizabethmiddle1819[,2], function(x)length(unique(x))))



Schools <- c("Greenock1718", "Greenock1819", "Central1718", "Central1819", "Williampenn1718", "Williampenn1819","Mountvernon1718", "MountVernon1819", "ElizabethMiddle1718", "Elizabethmiddle1819")

df <- data.frame(Schools,number.students)
df

```
####?data Identifying those students who were in elementary schools for 17-18 and middle school in 18-19

for (i in 1:nrow(combined.elementary1718)) {
  for(j in 1:nrow(combined.middle1819)){
    if(combined.elementary1718[[i,2]] == combined.middle1819[[j,2]]){
      combined.elementary1718$is.middle[i] <- 1
      combined.middle1819$is.middle[j] <- 1
    }
    
  }
  
}



```{r}
filtered1718 <- fread("filtered1718.csv")
filtered1819 <- fread("filtered1819.csv")
```

```{r}

analysis.data <- rbind(filtered1718, filtered1819)
number.students <- analysis.data %>%
  group_by(StudentID)%>%
  summarise(n())
nrow(number.students)
```

```{r}

#data for students who transitioned to middle in there second year
analysis.data <- filter(analysis.data, is.middle == 1)
number.transition.middle <- analysis.data %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.transition.middle)

#total student who did not transition to a middle school
nomiddle.transition <- student.analysis1[ !(student.analysis1$StudentID %in% analysis.data$StudentID),]

#student who moved to elementary from elementary in second year
nomiddle.transition.elementary <- nomiddle.transition[!(nomiddle.transition$SchoolName == "Elizabeth Forward Middle School")]
number.only.elementary <- nomiddle.transition.elementary %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.only.elementary)

#student who moved to middle from middle in second year
nomiddle.transition.middle <- nomiddle.transition[(nomiddle.transition$SchoolName == "Elizabeth Forward Middle School")]
number.only.middle <- nomiddle.transition.middle %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.only.middle)

```

```{r}

# preparing three sets of data: those who transitioned to middle, those who were in only elementary, those who were in only middle
analysis.data <- analysis.data %>%
  group_by(StudentID, TermName, Discipline) %>%
  summarise( rit.score = mean(TestRITScore))
nrow(analysis.data)

nomiddle.transition <- nomiddle.transition %>%
  group_by(StudentID, TermName, Discipline) %>%
  summarise( rit.score = mean(TestRITScore))
nrow(nomiddle.transition)

nomiddle.transition.elementary <- nomiddle.transition.elementary %>%
  group_by(StudentID, TermName, Discipline) %>%
  summarise( rit.score = mean(TestRITScore))
nrow(nomiddle.transition.elementary)

nomiddle.transition.middle <- nomiddle.transition.middle %>%
  group_by(StudentID, TermName, Discipline) %>%
  summarise( rit.score = mean(TestRITScore))


nrow(nomiddle.transition.middle)
```

```{r}
#merge demographic info
demographicInfo <- fread(input = "Studnet ID_IEP_Econ_Ethn_EF-hashed.csv")
names(demographicInfo)[1] <- names(analysis.data)[1]
analysis.data<- merge(analysis.data,demographicInfo)

number.transition.middle <- analysis.data %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.transition.middle)


#merge demographic info for students with no middle transition
demographicInfo <- fread(input = "Studnet ID_IEP_Econ_Ethn_EF-hashed.csv")
names(demographicInfo)[1] <- names(nomiddle.transition)[1]
nomiddle.transition<- merge(nomiddle.transition,demographicInfo)

#merge demographic info for students with no middle transition- Elementary only
demographicInfo <- fread(input = "Studnet ID_IEP_Econ_Ethn_EF-hashed.csv")
names(demographicInfo)[1] <- names(nomiddle.transition.elementary)[1]
nomiddle.transition.elementary<- merge(nomiddle.transition.elementary, demographicInfo)

number.only.elementary <- nomiddle.transition.elementary %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.only.elementary)



#merge demographic info for students with no middle transition- middle only
demographicInfo <- fread(input = "Studnet ID_IEP_Econ_Ethn_EF-hashed.csv")
names(demographicInfo)[1] <- names(nomiddle.transition.middle)[1]
nomiddle.transition.middle<- merge(nomiddle.transition.middle, demographicInfo, by = "StudentID")


number.only.middle <- nomiddle.transition.middle %>%
  group_by(StudentID)%>%
  summarise( total = n())
nrow(number.only.middle)


```


#### measuring growth rates in elementary and middle school for the same student

```{r, fig.width = 10, fig,height = 10}

#recofing factors for middle transition
analysis.data$term <- analysis.data$TermName
analysis.data <- mutate(analysis.data, term = recode_factor(analysis.data$term, `1` = "Fall 2017-2018", `2` = "Winter 2017-2018", `3` = "Spring 2017-2018", `4` = "Fall 2018-2019", `5` = "Winter 2018-2019", `6` = "Spring 2018-2019" ))
analysis.data$TermName <- factor (analysis.data$TermName, levels = c("Fall 2017-2018", "Winter 2017-2018", "Spring 2017-2018", "Fall 2018-2019", "Winter 2018-2019", "Spring 2018-2019"))

#converting levels into integers for geom_line()
g<- analysis.data$term
g <- as.integer(g)
analysis.data$term <- g

avg.race.maths.middle <- analysis.data[analysis.data$Discipline=="Mathematics"&analysis.data$Ethnicity%in%c("B","H","W"),] %>% 
        group_by(Ethnicity, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
avg.race.maths.middle 

```

```{r, fig.width=10, fig.height=10}
#recoding factor for only elementary
nomiddle.transition.elementary$term <- nomiddle.transition.elementary$TermName
nomiddle.transition.elementary <- mutate(nomiddle.transition.elementary, term = recode_factor(nomiddle.transition.elementary$term, `1` = "Fall 2017-2018", `2` = "Winter 2017-2018", `3` = "Spring 2017-2018", `4` = "Fall 2018-2019", `5` = "Winter 2018-2019", `6` = "Spring 2018-2019" ))
nomiddle.transition.elementary$TermName <- factor (nomiddle.transition.elementary$TermName, levels = c("Fall 2017-2018", "Winter 2017-2018", "Spring 2017-2018", "Fall 2018-2019", "Winter 2018-2019", "Spring 2018-2019"))


g<- nomiddle.transition.elementary$term
g <- as.integer(g)
nomiddle.transition.elementary$term <- g



race.maths.nomiddle.elementary <- nomiddle.transition.elementary[nomiddle.transition.elementary$Discipline=="Mathematics"&nomiddle.transition.elementary$Ethnicity%in%c("B","H","W"),] %>% 
        group_by(Ethnicity, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
race.maths.nomiddle.elementary

#plot for mathematics
ggplot(nomiddle.transition.elementary[nomiddle.transition.elementary$Discipline=="Mathematics"&nomiddle.transition.elementary$Ethnicity%in%c("B","H","W"),], 
       aes(TermName, y = rit.score, color = Ethnicity)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") + 
  geom_line(data = avg.race.maths.middle, aes(x = term, y = rit.score, color = Ethnicity), alpha = 1, size = 2)+
   geom_line(data = race.maths.nomiddle.elementary, aes(x = term, y = rit.score, color = Ethnicity), alpha = 1, size = 1, linetype ="dotted")+
  
  theme_bw()+
  theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : for those who transitioned to middle vs elementary")

```
```{r, fig.width=10, fig.height=10}
#recoding factor for no middle transition for only middle school
nomiddle.transition.middle$term <- nomiddle.transition.middle$TermName
nomiddle.transition.middle <- mutate(nomiddle.transition.middle, term = recode_factor(nomiddle.transition.middle$term, `1` = "Fall 2017-2018", `2` = "Winter 2017-2018", `3` = "Spring 2017-2018", `4` = "Fall 2018-2019", `5` = "Winter 2018-2019", `6` = "Spring 2018-2019" ))
nomiddle.transition.middle$TermName <- factor (nomiddle.transition.middle$TermName, levels = c("Fall 2017-2018", "Winter 2017-2018", "Spring 2017-2018", "Fall 2018-2019", "Winter 2018-2019", "Spring 2018-2019"))


g<- nomiddle.transition.middle$term
g <- as.integer(g)
nomiddle.transition.middle$term <- g



race.maths.nomiddle.middle <- nomiddle.transition.middle[nomiddle.transition.middle$Discipline=="Mathematics"&nomiddle.transition.middle$Ethnicity%in%c("B","H","W"),] %>% 
        group_by(Ethnicity, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
race.maths.nomiddle.middle


#plot for mathematics
ggplot(nomiddle.transition.middle[nomiddle.transition.middle$Discipline=="Mathematics"&nomiddle.transition.middle$Ethnicity%in%c("B","H","W"),], 
       aes(TermName, y = rit.score, color = Ethnicity)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") + 
  geom_line(data = avg.race.maths.middle, aes(x = term, y = rit.score, color = Ethnicity), alpha = 1, size = 2)+
   geom_line(data = race.maths.nomiddle.middle, aes(x = term, y = rit.score, color = Ethnicity), alpha = 1, size = 1, linetype ="dotted")+
  
  theme_bw()+
  theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : for those who transitioned to middle vs middle")
```







```{r, fig.width = 10, fig.height = 10}
#for those who transitioned to middle
avg.rit.econdis <- analysis.data[analysis.data$Discipline=="Mathematics"&analysis.data$EconDis%in%c("1","0", NA),] %>% 
        group_by(EconDis, term) %>% 
        summarise(avg.rit = mean(rit.score), count = n())

avg.rit.econdis$economy <- avg.rit.econdis$EconDis
avg.rit.econdis$EconDisLabel <- ifelse(avg.rit.econdis$EconDis==0,"Economically Disadvantaged",ifelse(avg.rit.econdis$EconDis==1,"Not Economically Disadvataged",NA))
avg.rit.econdis

#only for elementary school
avg.rit.econdis.elementary <- nomiddle.transition.elementary[nomiddle.transition.elementary$Discipline=="Mathematics"&nomiddle.transition.elementary$EconDis%in%c("1","0", NA),] %>% 
        group_by(EconDis, term) %>% 
        summarise(avg.rit = mean(rit.score), count = n())

avg.rit.econdis.elementary$economy <- avg.rit.econdis.elementary$EconDis
avg.rit.econdis.elementary$EconDisLabel <- ifelse(avg.rit.econdis.elementary$EconDis==0,"Economically Disadvantaged",ifelse(avg.rit.econdis.elementary$EconDis==1,"Not Economically Disadvataged",NA))


```

```{r, fig.width = 10, fig.height =10}

nomiddle.transition.elementary$economy <- nomiddle.transition.elementary$EconDis
nomiddle.transition.elementary$economy <- ifelse(nomiddle.transition.elementary$economy==0,"Economically Disadvantaged",ifelse(nomiddle.transition.elementary$economy==1,"Not Economically Disadvataged",NA))

ggplot(nomiddle.transition.elementary[nomiddle.transition.elementary$Discipline=="Mathematics"&nomiddle.transition.elementary$EconDis%in%c("1","0", NA),], 
       aes(x= TermName, y = rit.score, color = economy)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") +
  geom_line(data = avg.rit.econdis,mapping =  aes(x = term, y = avg.rit, color = EconDisLabel), alpha = 1, size =2) +
  geom_line(data = avg.rit.econdis.elementary,mapping =  aes(x = term, y = avg.rit, color = EconDisLabel), alpha = 1, size =1, linetype = "dotted")+
  theme_bw() +
   theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : economic disadvantage and transitioned to middle vs only elementary")

```

```{r, fig.width = 10, fig.height = 10}
avg.rit.econdis <- analysis.data[analysis.data$Discipline=="Mathematics"&analysis.data$EconDis%in%c("1","0", NA),] %>% 
        group_by(EconDis, term) %>% 
        summarise(avg.rit = mean(rit.score), count = n())

avg.rit.econdis$economy <- avg.rit.econdis$EconDis
avg.rit.econdis$EconDisLabel <- ifelse(avg.rit.econdis$EconDis==0,"Economically Disadvantaged",ifelse(avg.rit.econdis$EconDis==1,"Not Economically Disadvataged",NA))
avg.rit.econdis

#only for middle
avg.rit.econdis.middle <- nomiddle.transition.middle[nomiddle.transition.middle$Discipline=="Mathematics"&nomiddle.transition.middle$EconDis%in%c("1","0", NA),] %>% 
        group_by(EconDis, term) %>% 
        summarise(avg.rit = mean(rit.score), count = n())

avg.rit.econdis.middle$economy <- avg.rit.econdis.middle$EconDis
avg.rit.econdis.middle$EconDisLabel <- ifelse(avg.rit.econdis.middle$EconDis==0,"Economically Disadvantaged",ifelse(avg.rit.econdis.elementary$EconDis==1,"Not Economically Disadvataged",NA))


```

```{r, fig.width = 10, fig.height =10}

# only for middle
nomiddle.transition.middle$economy <- nomiddle.transition.middle$EconDis
nomiddle.transition.middle$economy <- ifelse(nomiddle.transition.middle$economy==0,"Economically Disadvantaged",ifelse(nomiddle.transition.middle$economy==1,"Not Economically Disadvataged",NA))

ggplot(nomiddle.transition.middle[nomiddle.transition.middle$Discipline=="Mathematics"&nomiddle.transition.middle$EconDis%in%c("1","0", NA),], 
       aes(x= TermName, y = rit.score, color = economy)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") +
  geom_line(data = avg.rit.econdis,mapping =  aes(x = term, y = avg.rit, color = EconDisLabel), alpha = 1, size =2) +
  geom_line(data = avg.rit.econdis.middle,mapping =  aes(x = term, y = avg.rit, color = EconDisLabel), alpha = 1, size =1, linetype = "dotted")+
  theme_bw() +
   theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : economic disadvantage and transitioned to middle vs only middle")

```

```{r, fig.width=10, fig.height=10}
#changing the race for minoritised vs non-minoritised
analysis.data$new.race <- ifelse(analysis.data$Ethnicity %in% c("B", "H", "O"),0,ifelse(analysis.data$Ethnicity %in% c("W", "A"),1,NA))

analysis.data$race <- ifelse(analysis.data$new.race == 0,"Minoritised",ifelse(analysis.data$new.race == 1, "Non-Minoritised",NA))


avg.race.maths <- analysis.data[analysis.data$Discipline=="Mathematics",] %>% 
        group_by(race, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
avg.race.maths

#changing the race for minoritised vs non-minoritised for only elementary
nomiddle.transition.elementary$new.race <- ifelse(nomiddle.transition.elementary$Ethnicity %in% c("B", "H", "O"),0,ifelse(nomiddle.transition.elementary$Ethnicity %in% c("W", "A"),1,NA))

nomiddle.transition.elementary$race <- ifelse(nomiddle.transition.elementary$new.race == 0,"Minoritised",ifelse(nomiddle.transition.elementary$new.race == 1, "Non-Minoritised",NA))


avg.race.maths.onlyelem <- nomiddle.transition.elementary[analysis.data$Discipline=="Mathematics",] %>% 
        group_by(race, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
avg.race.maths.onlyelem 


#plot for mathematics
ggplot(nomiddle.transition.elementary[nomiddle.transition.elementary$Discipline=="Mathematics",], aes(TermName, y = rit.score, color = race)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") + 
  geom_line(data = avg.race.maths, aes(x = term, y = rit.score, color = race), alpha = 1, size = 2)+
  geom_line(data = avg.race.maths.onlyelem, aes(x = term, y = rit.score, color = race), alpha = 1, size = 1, linetype ="dotted")+
  theme_bw()+
  theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : for minoritised vs non-minoritised and Middle transition vs only elementary")


```



```{r, fig.width=10, fig.height=10}
#changing the race for minoritised vs non-minoritised
analysis.data$new.race <- ifelse(analysis.data$Ethnicity %in% c("B", "H", "O"),0,ifelse(analysis.data$Ethnicity %in% c("W", "A"),1,NA))

analysis.data$race <- ifelse(analysis.data$new.race == 0,"Minoritised",ifelse(analysis.data$new.race == 1, "Non-Minoritised",NA))


avg.race.maths <- analysis.data[analysis.data$Discipline=="Mathematics",] %>% 
        group_by(race, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
avg.race.maths

#changing the race for minoritised vs non-minoritised only for middle
nomiddle.transition.middle$new.race <- ifelse(nomiddle.transition.middle$Ethnicity %in% c("B", "H", "O"),0,ifelse(nomiddle.transition.middle$Ethnicity %in% c("W", "A"),1,NA))

nomiddle.transition.middle$race <- ifelse(nomiddle.transition.middle$new.race == 0,"Minoritised",ifelse(nomiddle.transition.middle$new.race == 1, "Non-Minoritised",NA))


avg.race.maths.onlymiddle <- nomiddle.transition.middle[analysis.data$Discipline=="Mathematics",] %>% 
        group_by(race, term) %>% 
        summarise(rit.score = mean(rit.score), count = n())
avg.race.maths.onlymiddle


#plot for mathematics
ggplot(nomiddle.transition.middle[nomiddle.transition.middle$Discipline=="Mathematics",], aes(TermName, y = rit.score, color = race)) + 
  geom_line(aes(group = StudentID), alpha = .3, color = "yellow") + 
  geom_line(data = avg.race.maths, aes(x = term, y = rit.score, color = race), alpha = 1, size = 2)+
  geom_line(data = avg.race.maths.onlymiddle, aes(x = term, y = rit.score, color = race), alpha = 1, size = 1, linetype ="dotted")+
  theme_bw()+
  theme(axis.text.x = element_text(angle = 60, hjust =1, vjust =1)) +
  ggtitle("Growth Mathematics : for minoritised vs non-minoritised and Middle transition vs only middle")


```


```{r}
#number of students for three groups by ethnicity
setDT(analysis.data)
analysis.data[,.N, by=  .(StudentID,Ethnicity)][,.N, by = Ethnicity]

setDT(nomiddle.transition.elementary)
nomiddle.transition.elementary[,.N, by=  .(StudentID,Ethnicity)][,.N, by = Ethnicity]

setDT(nomiddle.transition.middle)
nomiddle.transition.middle[,.N, by=  .(StudentID,Ethnicity)][,.N, by = Ethnicity]

```