YS-Aug29.Rmd

---
title: "Hetero Analysis"
author: "Yicheng Shen"
date: "26/03/2022"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(mosaic)
ggplot2::theme_set(ggplot2::theme_bw())
knitr::opts_chunk$set(out.width = "100%", fig.align = 'center')
```


```{r}
res <- read.csv('res/res_binary.csv')   
res2 <- read.csv('res/htelogit_tau50.csv')   
res3 <- read.csv('res/ab_tau50.csv')   

modelname = rep("htelogit_tau50", nrow(res2))
col.group = rep("Bayesian", nrow(res2))
pch.group = rep("HTE model", nrow(res2))

res2 = cbind(res2[, 1:3], modelname, col.group, pch.group, res2[, 4:ncol(res2)])
res2$I.2 = NA


modelname = rep("ablogit_tau50", nrow(res3))
col.group = rep("Bayesian", nrow(res3))
pch.group = rep("HTE model", nrow(res3))

res3 = cbind(res3[, 1:3], modelname, col.group, pch.group, res3[, 4:ncol(res3)])
res3$I.2 = NA

res_all = rbind(res, res2, res3)
```




```{r}
result = res_all %>% 
  mutate(modelname = factor(modelname, levels = c("ivfe", "ivre", "ctevaguelogit","ctebeta", "htevaguelogit", "htelogit_tau50", "htebeta", "ablogit", "ablogit_tau50")))

n_distinct(result$MA.id)
nrow(result) # 9 model result per study

table((result %>% distinct(MA.id, .keep_all = TRUE))$event)
```


```{r}
result = result %>% 
  mutate(Significant_num = 1-as.numeric(0 >= low & 0 <= up)) %>% 
  mutate(Significant = ifelse(Significant_num == "1", "significant", "non-significant"))

result$Significant <- factor(result$Significant, levels = c("significant", "non-significant")) 

# result %>% ggplot() + geom_ribbon(aes(x = MA.id, ymin = low, ymax = up)) + facet_wrap(~modelname)
```


## analyze seperate outcome type

```{r}
result_com = result %>% filter(event %in% c("common"))
n_distinct(result_com$MA.id)
result_if = result %>% filter(event %in% c("infrequent"))
n_distinct(result_if$MA.id)
result_ra = result %>% filter(event %in% c("rare"))
n_distinct(result_ra$MA.id)
result_vra = result %>% filter(event %in% c("very rare"))
n_distinct(result_vra$MA.id)
```



```{r}
create_table <- function(result){

modelnames <- unique(result$modelname)

table_list <- list()

for (i in 1:(length(modelnames)-1)) {
  for (j in (i+1):length(modelnames)) {
    model1 <- modelnames[i]
    model2 <- modelnames[j]
    
    # Subset the data for the current combination of modelnames
    subset_data <- result[result$modelname %in% c(model1, model2), ]
    
    # Create a 2x2 contingency table
    contingency_table <- table(subset_data$Significant[subset_data$modelname == model1],
                               subset_data$Significant[subset_data$modelname == model2])
    
    # Add the table to the list
    table_list[[paste(model1, model2, sep = "_vs_")]] <- contingency_table
  }
}

# Print the tables
for (table_name in names(table_list)) {
  cat("Table:", table_name, "\n")
  print(table_list[[table_name]])
  cat("\n")
}
}
```


```{r}
# create_table(result_com)
# create_table(result_if)
# create_table(result_ra)
# create_table(result_vra)
```



# ivfe_vs_ivre

```{r}
table(result$Significant[result$modelname == "ivfe"],
                        result$Significant[result$modelname == "ivre"])

proportions(table(result$Significant[result$modelname == "ivfe"],
                        result$Significant[result$modelname == "ivre"]))* 100

cor( (result %>% filter(modelname %in% c("ivfe")))$est,
     (result %>% filter(modelname %in% c("ivre")))$est )

cor( (result %>% filter(modelname %in% c("ivfe")))$se,
     (result %>% filter(modelname %in% c("ivre")))$se )
```

```{r}
# differ inference (1381)
# result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
#   group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
#   filter(Significant_sum == 1)

# those 1326
expected_id = (result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ivfe"))$MA.id

# those 55
strange_id = (result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ivre"))$MA.id
```


```{r}
result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, shape = event, color = difference < 0), size = 1) +
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 0.5, fill = "gray") + 
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname) + labs(color = "ivre has lower est")


result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% strange_id) %>%
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = difference < 0), size = 1) + facet_wrap(~modelname) +
  labs(color = "ivre has lower est")

result_strange <- result %>% filter(MA.id %in% strange_id) 
prop.table(table(result_strange$event))

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% strange_id))$est,
     (result %>% filter(modelname %in% c("ivre"), MA.id %in% strange_id))$est )

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% strange_id))$se,
     (result %>% filter(modelname %in% c("ivre"), MA.id %in% strange_id))$se )
```


```{r}
result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 1, fill = "gray") +
  geom_point(aes(index, est, color = event), size = 0.02) +
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + geom_col(aes(index, width))+ facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% expected_id))$est,
     (result %>% filter(modelname %in% c("ivre"), MA.id %in% expected_id))$est )

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% expected_id))$se,
     (result %>% filter(modelname %in% c("ivre"), MA.id %in% expected_id))$se )


result_expected <- result %>% filter(MA.id %in% expected_id) 
prop.table(table(result_expected$event))
```



```{r}
result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.5)

result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.2)


result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.5)

result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.2)


result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.3)

result %>% filter(modelname %in% c("ivfe", "ivre")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.3)
```




# ctevaguelogit_vs_htevaguelogit


```{r}
table(result$Significant[result$modelname == "ctevaguelogit"],
                        result$Significant[result$modelname == "htevaguelogit"])

proportions(table(result$Significant[result$modelname == "ctevaguelogit"],
                        result$Significant[result$modelname == "htevaguelogit"]))*100

cor( (result %>% filter(modelname %in% c("ctevaguelogit")))$est,
     (result %>% filter(modelname %in% c("htevaguelogit")))$est )

cor( (result %>% filter(modelname %in% c("ctevaguelogit")))$se,
     (result %>% filter(modelname %in% c("htevaguelogit")))$se )

# differ inference 4449
# result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
#   group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
#   filter(Significant_sum == 1)


# those 4347
expected_id = (result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ctevaguelogit"))$MA.id

# those 102
strange_id = (result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "htevaguelogit"))$MA.id
```



```{r}
result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_hline(yintercept = 0, linewidth = 1) + 
  geom_point(aes(index, est, color = event), size = 0.5) +
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 0.5, fill = "gray") + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(MA.id) %>% 
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = difference < 0), size = 0.5) + facet_wrap(~modelname) + labs(color = "htevaguelogit lower")

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.3) + facet_wrap(~modelname)

cor( (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% strange_id))$est,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% strange_id))$est )

cor( (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% strange_id))$se,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% strange_id))$se )


result_strange <- result %>% filter(MA.id %in% strange_id) 
prop.table(table(result_strange$event))
```

```{r}
result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
    group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = event)) 
```


```{r}
result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 1, fill = "gray") +
  geom_point(aes(index, est, color = event), size = 0.02) +
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + geom_col(aes(index, width))+ facet_wrap(~modelname)


result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


cor( (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% expected_id))$est,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% expected_id))$est )

cor( (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% expected_id))$se,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% expected_id))$se )


result_expected <- result %>% filter(MA.id %in% expected_id) 
prop.table(table(result_expected$event))
```

```{r}
result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 1)

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 1)


result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.5)

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.2)



result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.3)

result %>% filter(modelname %in% c("ctevaguelogit", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.3)
```


# ctebeta_vs_hetbeta


```{r}
missing.MA.id = (result %>% filter(modelname == "htebeta") %>% filter(is.na(est)))$MA.id
result_narm = result %>% filter(! MA.id %in% missing.MA.id)

  
table(result_narm$Significant[result_narm$modelname == "ctebeta"],
                        result_narm$Significant[result_narm$modelname == "htebeta"])

proportions(table(result_narm$Significant[result_narm$modelname == "ctebeta"],
                        result_narm$Significant[result_narm$modelname == "htebeta"])) * 100

cor( (result_narm %>% filter(modelname %in% c("ctebeta")))$est,
     (result_narm %>% filter(modelname %in% c("htebeta")))$est)

cor( (result_narm %>% filter(modelname %in% c("ctebeta")))$se,
     (result_narm %>% filter(modelname %in% c("htebeta")))$se )

# differ inference 
# result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
#   group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
#   filter(Significant_sum == 1)


# those 6913
expected_id = (result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ctebeta"))$MA.id

# those 18
strange_id = (result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "htebeta"))$MA.id
```

```{r}
result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, shape = event, color = difference < 0), size = 1) +
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 0.5, fill = "gray") + facet_wrap(~modelname)


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(MA.id) %>% 
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = difference < 0), size = 0.5) + facet_wrap(~modelname) + labs(color = "htebeta lower")

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.3) + facet_wrap(~modelname)

cor( (result_narm %>% filter(modelname %in% c("ctebeta"), MA.id %in% strange_id))$est,
     (result_narm %>% filter(modelname %in% c("htebeta"), MA.id %in% strange_id))$est )

cor( (result_narm %>% filter(modelname %in% c("ctebeta"), MA.id %in% strange_id))$se,
     (result_narm %>% filter(modelname %in% c("htebeta"), MA.id %in% strange_id))$se )


result_strange <- result_narm %>% filter(MA.id %in% strange_id) 
prop.table(table(result_strange$event))
```


```{r}
result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
    group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

weird.id = (result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0))$MA.id

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  mutate(weird = ifelse(MA.id %in% weird.id, "HTE smaller CI", "HTE larger CI")) %>% filter(weird == "HTE smaller CI")


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = event)) 
```



```{r}
result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 1, fill = "gray") +
  geom_point(aes(index, est, color = event), size = 0.02) +
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname)


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + geom_col(aes(index, width))+ facet_wrap(~modelname)


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


cor( (result_narm %>% filter(modelname %in% c("ctebeta"), MA.id %in% expected_id))$est,
     (result_narm %>% filter(modelname %in% c("htebeta"), MA.id %in% expected_id))$est )

cor( (result_narm %>% filter(modelname %in% c("ctebeta"), MA.id %in% expected_id))$se,
     (result_narm %>% filter(modelname %in% c("htebeta"), MA.id %in% expected_id))$se )


result_expected <- result_narm %>% filter(MA.id %in% expected_id) 
prop.table(table(result_expected$event))
```


```{r}
result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 1)

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 1)


result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.5)

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.2)


# all
result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.3)

result_narm %>% filter(modelname %in% c("ctebeta", "htebeta")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.3)
```


# ctevaguelogit_vs_ivfe


```{r}
table(result$Significant[result$modelname == "ivfe"],
        result$Significant[result$modelname == "ctevaguelogit"])

proportions(table(result$Significant[result$modelname == "ivfe"],
        result$Significant[result$modelname == "ctevaguelogit"])) * 100


cor( (result_narm %>% filter(modelname %in% c("ivfe")))$est,
     (result_narm %>% filter(modelname %in% c("ctevaguelogit")))$est)

cor( (result_narm %>% filter(modelname %in% c("ivfe")))$se,
     (result_narm %>% filter(modelname %in% c("ctevaguelogit")))$se )


# 
expected_id = (result_narm %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ivfe"))$MA.id

#
strange_id = (result_narm %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ctevaguelogit"))$MA.id
```

```{r}
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_hline(yintercept = 0, linewidth = 1) + 
  geom_point(aes(index, est, color = event), size = 0.5) +
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 0.5, fill = "gray") + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(MA.id) %>% 
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = difference < 0), size = 0.5) + facet_wrap(~modelname) + labs(color = "ctevaguelogit lower")

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.3) + facet_wrap(~modelname)

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% strange_id))$est,
     (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% strange_id))$est )

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% strange_id))$se,
     (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% strange_id))$se )


result_strange <- result %>% filter(MA.id %in% strange_id) 
prop.table(table(result_strange$event))
```

```{r}
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    filter(MA.id %in% strange_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

weird.id = (result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0))$MA.id

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  mutate(weird = ifelse(MA.id %in% weird.id, "CTE smaller CI", "CTE larger CI")) %>% filter(weird == "CTE smaller CI")


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = event)) 

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = diff_width < 0)) 
```


```{r}
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 1, fill = "gray") +
  geom_point(aes(index, est, color = event), size = 0.5) +
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + geom_col(aes(index, width))+ facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% expected_id))$est,
     (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% expected_id))$est )

cor( (result %>% filter(modelname %in% c("ivfe"), MA.id %in% expected_id))$se,
     (result %>% filter(modelname %in% c("ctevaguelogit"), MA.id %in% expected_id))$se )


result_expected <- result %>% filter(MA.id %in% expected_id) 
prop.table(table(result_expected$event))
```



```{r}
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.8)

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.8)


result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.8)

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.8)


# all
result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.3)

result %>% filter(modelname %in% c("ivfe", "ctevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.3)
```


# htevaguelogit_vs_ivre


```{r}
table(result$Significant[result$modelname == "ivre"],
        result$Significant[result$modelname == "htevaguelogit"])

proportions(table(result$Significant[result$modelname == "ivre"],
        result$Significant[result$modelname == "htevaguelogit"]))*100


cor( (result %>% filter(modelname %in% c("ivre")))$est,
     (result %>% filter(modelname %in% c("htevaguelogit")))$est)

cor( (result %>% filter(modelname %in% c("ivre")))$se,
     (result %>% filter(modelname %in% c("htevaguelogit")))$se )


# 
expected_id = (result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "ivre"))$MA.id

#
strange_id = (result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(Significant_sum = sum(Significant_num)) %>% 
  filter(Significant_sum == 1, Significant == "significant", modelname == "htevaguelogit"))$MA.id
```

```{r}
result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_hline(yintercept = 0, linewidth = 1) + 
  geom_point(aes(index, est, color = event), size = 0.5) +
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 0.5, fill = "gray") + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(MA.id) %>% 
  mutate(difference = diff(est)) %>% 
  ungroup() %>% 
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = difference < 0), size = 0.5) + facet_wrap(~modelname) + labs(color = "ctevaguelogit lower")

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.3) + facet_wrap(~modelname)

cor( (result %>% filter(modelname %in% c("ivre"), MA.id %in% strange_id))$est,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% strange_id))$est )

cor( (result %>% filter(modelname %in% c("ivre"), MA.id %in% strange_id))$se,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% strange_id))$se )


result_strange <- result %>% filter(MA.id %in% strange_id) 
prop.table(table(result_strange$event))
```

```{r}
result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)
result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    filter(MA.id %in% strange_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

weird.id = (result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0))$MA.id

# result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  # mutate(weird = ifelse(MA.id %in% weird.id, "htevaguelogit smaller CI", "htevaguelogit larger CI")) %>% filter(weird == "htevaguelogit smaller CI")


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = event)) 

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    group_by(MA.id) %>% reframe(event = event, diff_width = diff(width)) %>% distinct() %>% 
  ggplot() + geom_point(aes(MA.id, diff_width, color = diff_width < 0)) 
```


```{r}
result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_ribbon(aes(x = index, ymin = low, ymax = up), alpha = 1, fill = "gray") +
  geom_point(aes(index, est, color = event), size = 0.5) +
  geom_hline(yintercept = 0, linewidth = 1) + facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
    filter(MA.id %in% expected_id) %>% group_by(MA.id) %>% summarize(diff(width)) %>% filter(`diff(width)` < 0)

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + geom_col(aes(index, width))+ facet_wrap(~modelname)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = event), size = 0.5) + facet_wrap(~modelname)


cor( (result %>% filter(modelname %in% c("ivre"), MA.id %in% expected_id))$est,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% expected_id))$est )

cor( (result %>% filter(modelname %in% c("ivre"), MA.id %in% expected_id))$se,
     (result %>% filter(modelname %in% c("htevaguelogit"), MA.id %in% expected_id))$se )


result_expected <- result %>% filter(MA.id %in% expected_id) 
prop.table(table(result_expected$event))
```





```{r}
result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.8)

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% strange_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.8)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.8)

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  filter(MA.id %in% expected_id) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.8)


result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>%
  ungroup() %>% 
  ggplot() + 
  geom_point(aes(index, est, color = modelname), size = 0.8)

result %>% filter(modelname %in% c("ivre", "htevaguelogit")) %>% 
  group_by(MA.id) %>% mutate(avg_est = mean(est)) %>% arrange(avg_est) %>%
  group_by(modelname) %>%
  mutate(index = row_number()) %>% ungroup() %>% 
  group_by(MA.id) %>% mutate(diff_est = diff(est)) %>% ungroup() %>% 
  ggplot() + geom_point(aes(index, diff_est, color = event), size = 0.8)
```

# correlation of tau by different model