Assessing the Intuitiveness of Qualitative Contribution Relationships in Goal Models: an Exploratory Experiment (ESEM 2017)

Abstract

This page accompanies the ESEM 2017 short paper submission. The data, redacted to include information only relevant to this study and remove personally identifying information can be found here and here, published here in compliance to clause “non-identifying responses and aggregations thereof will appear in the PI’s research report and potential publications of this research” of informed consent.

fLoc <- "http://www.yorku.ca/liaskos/Papers/ESEM2017/Data/"
#fLoc <- "..\\Session3\\"
# Read the data
dL <- read.csv(paste0(fLoc,"LL.csv"),header = TRUE,stringsAsFactors=FALSE)
dW <- read.csv(paste0(fLoc,"WW.csv"),header = TRUE,stringsAsFactors=FALSE)
sol <- read.csv(paste0(fLoc,"Normative.csv"),header = TRUE,stringsAsFactors=FALSE)

labs = c("FS","PS","N","PD","FD")
levs = c(1,2,3,4,5)

Spath <- "http://www.yorku.ca/liaskos/Papers/ESEM2017/Images/S/"
Wpath <- "http://www.yorku.ca/liaskos/Papers/ESEM2017/Images/W/"

dLO <- dL
dWO <- dW

#
# Clean - up 
#

# Incomplete responses
dL <- dL[dL$Status == "Complete",]
dW <- dW[dW$Status == "Complete",]

# L: 64, 71: failed at least two tests
# L: 55, 73: failed one tests
dL <- subset(dL, !(Response.ID %in% c(71,64)))



# W: 45, 53, 41: failed at least two tests
# W: 52, 54: failed one test
dW <- subset(dW, !(Response.ID %in% c(45,53,41)))

# Coding misconfiguration in Surveygizmo.
dL$C.18 <- dL$C.18 - 5
dW$C.18 <- dW$C.18 - 5

Sample Characteristics

A total of 41 persons showed up for the experiment, 21 assigned to the symbolic instrument and 20 to the textual. Out of these 5 are filtered out due to low performance in questions testing their comprehension of concepts presented in the videos and 1 due to an incomplete response. The remaining 35 cases, , 18 in the symbolic and 17 in the textual instrument, are used for the analysis. They are 26 females and 9 males, their ages are predominantly 18-29 and their field of study primarily Business and Economics.

Descriptives

dLBU <- dL
dWBU <- dW


# Create the factors first
for (i in 1:20){
  col = paste0("C.",i)
  dL[,col] <- factor(dL[,col],levels = levs,labels = labs)
  dW[,col] <- factor(dW[,col],levels = levs,labels = labs)
}

Descriptives: Positive Contributions

Models are annotated with the column name by which they represented in the data file (C.1, C.2 etc).

par(mfrow=c(10,4), mar=c(2,2,2,2), xpd=NA, mgp=c(0,0,0), oma=c(0,0,0,0), ann=F)


# Display the positive contributions
for (i in 1:10){
  plot.new()
  
  sf<-sol[i,"Sfile"] 
  wf<-sol[i,"Wfile"]
  
  myurlS <- paste0(Spath,sf)
  myurlW <- paste0(Wpath,wf)
  
  col = paste0("C.",i)

  img<-readPNG(getURLContent(myurlS))
  usr<-par("usr")
  rasterImage(img, usr[1], usr[3], usr[2], usr[4])
  text(0.1,0.95,col,cex = 2)
  
  
  stat1  = paste0("Mean = ", round(3-mean(dLBU[,col],na.rm = TRUE),2))
  stat2  = paste0("Stdev = ", round(sd(dLBU[,col],na.rm = TRUE),2))
  barplot(table(dL[,col]),main = paste0(stat1,"\n",stat2))
  
  plot.new()
  
  img<-readPNG(getURLContent(myurlW))
  usr<-par("usr")
  rasterImage(img, usr[1], usr[3], usr[2], usr[4])
  text(0.1,0.95,col,cex = 2)
  
  
  stat1  = paste0("Mean = ", round(3-mean(dWBU[,col],na.rm = TRUE),2))
  stat2  = paste0("Stdev = ", round(sd(dWBU[,col],na.rm = TRUE),2))
  barplot(table(dW[,col]),main = paste0(stat1,"\n",stat2))

}

Descriptives: Negative Contributions

Models are annotated with the column name by which they represented in the data file (C.1, C.2 etc).

par(mfrow=c(10,4), mar=c(2,2,2,2), xpd=NA, mgp=c(0,0,0), oma=c(0,0,0,0), ann=F)


# Display the negative contributions
for (i in 11:20){
  plot.new()
  
  sf<-sol[i,"Sfile"] 
  wf<-sol[i,"Wfile"]
  
  myurlS <- paste0(Spath,sf)
  myurlW <- paste0(Wpath,wf)
  
  col = paste0("C.",i)

  img<-readPNG(getURLContent(myurlS))
  usr<-par("usr")
  rasterImage(img, usr[1], usr[3], usr[2], usr[4])
  text(0.1,0.95,col,cex = 2)
  
  
  stat1  = paste0("Mean = ", round(3-mean(dLBU[,col],na.rm = TRUE),2))
  stat2  = paste0("Stdev = ", round(sd(dLBU[,col],na.rm = TRUE),2))
  barplot(table(dL[,col]),main = paste0(stat1,"\n",stat2))
  
  plot.new()
  
  img<-readPNG(getURLContent(myurlW))
  usr<-par("usr")
  rasterImage(img, usr[1], usr[3], usr[2], usr[4])
  text(0.1,0.95,col,cex = 2)

  stat1  = paste0("Mean = ", round(3-mean(dWBU[,col],na.rm = TRUE),2))
  stat2  = paste0("Stdev = ", round(sd(dWBU[,col],na.rm = TRUE),2))
  barplot(table(dW[,col]),main = paste0(stat1,"\n",stat2))


}

Agreement Analysis

To measure the distance between participant responses we first map FD, PD, N, PS, FS (their actual responses) to the interval scale \([1,5]\). Then for each of the twenty scenarios and for each group (symbolic vs. textual) we perform all pair-wise comparisons between participant responses \(r_i\) and \(r_j\), \(i,j = 1\ldots N, i\neq j\) to calculate the normalized distance \(|r_i - r_j|/4\); the average of all these \(N(N-1)/2\) distances is considered, \(N\) being the number of participants for each group. The resulting set consists of \(2\) (groups) \(\times\) \(20\) (exercises) \(= 40\) data points each expressing level of total distance between the ratings of every pair of participants.

BoxPlots

# The distance function
sim <- function(x) {
  n <- length(x)
  sim <- matrix(0, nrow = n,ncol = n)
  for (i in 1:n){
    for (j in i:n){
      if (i!=j){
        # Absolute distance between ech pair of values
        # divided by the max absolute distance (5-1) for normalization.
        sim[i,j] = abs(x[i]-x[j])/4
      }
    }
  }
  return ( sum(sim)/(n*(n-1)/2) )
}


agg <- function(x) {return(sim(x))}

Agreement wrt. Sign and Intensity

# Load the data from before they became factors
dLsd <- dLBU
dWsd <- dWBU


AppL <- c(agg(dLsd$C.1), agg(dLsd$C.2), agg(dLsd$C.3), agg(dLsd$C.4), agg(dLsd$C.5)) 
ApL <- c(agg(dLsd$C.6), agg(dLsd$C.7), agg(dLsd$C.8), agg(dLsd$C.9), agg(dLsd$C.10))
AmL <- c(agg(dLsd$C.11), agg(dLsd$C.12), agg(dLsd$C.13), agg(dLsd$C.14), agg(dLsd$C.15)) 
AmmL <- c(agg(dLsd$C.16), agg(dLsd$C.17), agg(dLsd$C.18), agg(dLsd$C.19), agg(dLsd$C.20))

AppW <- c(agg(dWsd$C.1), agg(dWsd$C.2), agg(dWsd$C.3), agg(dWsd$C.4), agg(dWsd$C.5)) 
ApW <- c(agg(dWsd$C.6), agg(dWsd$C.7), agg(dWsd$C.8), agg(dWsd$C.9), agg(dWsd$C.10))
AmW <- c(agg(dWsd$C.11), agg(dWsd$C.12), agg(dWsd$C.13), agg(dWsd$C.14), agg(dWsd$C.15)) 
AmmW <- c(agg(dWsd$C.16), agg(dWsd$C.17), agg(dWsd$C.18), agg(dWsd$C.19), agg(dWsd$C.20))

agrD <- rbind(
      data.frame(Group = "Symbolic", Type = "++", Sign = "Pos",Intensity = "Double", Agr = AppL),
      data.frame(Group = "Symbolic", Type = "+", Sign = "Pos", Intensity = "Single", Agr = ApL),
      data.frame(Group = "Symbolic", Type = "-", Sign = "Neg", Intensity = "Single", Agr = AmL),
      data.frame(Group = "Symbolic", Type = "--", Sign = "Neg", Intensity = "Double", Agr = AmmL),

      data.frame(Group = "Textual",Type = "++",Sign = "Pos",Intensity = "Double", Agr = AppW),
      data.frame(Group = "Textual",Type = "+",Sign = "Pos",Intensity = "Single", Agr = ApW),
      data.frame(Group = "Textual",Type = "-",Sign = "Neg",Intensity = "Single", Agr = AmW),
      data.frame(Group = "Textual",Type = "--",Sign = "Neg",Intensity = "Double", Agr = AmmW)
      )

agrDBU <- agrD
agrD$Intensity = factor(agrD$Intensity,levels(agrD$Intensity)[c(2,1)])

asib1 <- ggplot(data = agrD,aes(x=Group,y=Agr,fill=Sign)) + geom_boxplot() + ylab("Total Distance") + scale_fill_manual(values = c("#5599FF","#FF5555"))
asib2 <- ggplot(data = agrD,aes(x=Group,y=Agr,fill=Intensity)) + geom_boxplot() + ylab("Total Distance")

grid.arrange(asib1,asib2,ncol = 2)

There is an obvious difference between positive and negative contributions, the latter yielding more disagreement. An effect can also be seen in intensity yet more subtle. Interestingly, a greater level of agreement appears to emerge in symbolic representations, especially in positive contributions.

Agreement wrt. Sign and Origin Satisfaction

In these two graphs we compare models in which the origin has a positive contribution with those whose origin has a negative one. Responses associated with models in which the satisfaction of the origin is “no information” (N) are omitted.

While a possible effect of the group (symbolic vs. textual) is very subtle, there is a clearer effect can be seen in the figure on the right: denial of origin is always a source of disagreement. The difference is more pronounced in the positive contributions where the disagreement is nevertheless generally less.

# Load the data from before they became factors
dLsd <- dLBU
dWsd <- dWBU


ApSatL <- c(agg(dLsd$C.1), agg(dLsd$C.2), agg(dLsd$C.6), agg(dLsd$C.7)) 
ApDenL <- c(agg(dLsd$C.4), agg(dLsd$C.5), agg(dLsd$C.9), agg(dLsd$C.10))
AmSatL <- c(agg(dLsd$C.11), agg(dLsd$C.12), agg(dLsd$C.16), agg(dLsd$C.17)) 
AmDenL <- c(agg(dLsd$C.14), agg(dLsd$C.15), agg(dLsd$C.19), agg(dLsd$C.20))

ApSatW <- c(agg(dWsd$C.1), agg(dWsd$C.2), agg(dWsd$C.6), agg(dWsd$C.7)) 
ApDenW <- c(agg(dWsd$C.4), agg(dWsd$C.5), agg(dWsd$C.9), agg(dWsd$C.10))
AmSatW <- c(agg(dWsd$C.11), agg(dWsd$C.12), agg(dWsd$C.16), agg(dWsd$C.17)) 
AmDenW <- c(agg(dWsd$C.14), agg(dWsd$C.15), agg(dWsd$C.19), agg(dWsd$C.20))


agrDSat <- rbind(
      data.frame(Group = "Symbolic", Sign = "Pos", Origin = "Satisfied", Agr = ApSatL),
      data.frame(Group = "Symbolic", Sign = "Pos", Origin = "Denied", Agr = ApDenL),
      data.frame(Group = "Symbolic", Sign = "Neg", Origin = "Satisfied", Agr = AmSatL),
      data.frame(Group = "Symbolic", Sign = "Neg", Origin = "Denied", Agr = AmDenL),

      data.frame(Group = "Textual", Sign = "Pos", Origin = "Satisfied", Agr = ApSatW),
      data.frame(Group = "Textual", Sign = "Pos", Origin = "Denied", Agr = ApDenW),
      data.frame(Group = "Textual", Sign = "Neg", Origin = "Satisfied", Agr = AmSatW),
      data.frame(Group = "Textual", Sign = "Neg", Origin = "Denied", Agr = AmDenW)
      )



asob1 <- ggplot(data = agrDSat,aes(x=Group,y=Agr,fill=Origin)) + geom_boxplot() + ylab("Total Distance") + scale_fill_manual(values = c("#5599FF","#FF5555"))

asob2 <- ggplot(data = agrDSat,aes(x=Sign,y=Agr,fill=Origin)) + geom_boxplot() + ylab("Total Distance") + scale_fill_manual(values = c("#5599FF","#FF5555"))

grid.arrange(asob1,asob2,ncol = 2)

#
#
# D A T A     P R E P A R A T I O N
#   F O R     A C C U R A C Y 
#



dLNom <- dLBU
dLNom$Group <- "Symbolic"
dLNom$RID<- paste0("L",dLNom$Response.ID)

dWNom <- dWBU
dWNom$Group <- "Textual"
dWNom$RID<- paste0("W",dWNom$Response.ID)

dLNomSign <- dLNom
dWNomSign <- dWNom


#
# Data coded reverselly 1-FS, 2-PS, etc.
# Thus, distances below are effectivelly: 
#
#      o b  s    -    n o r m
#
# Positive distance = participants overestimate satisfaction
# Negative distance = participants underestimate satisfaction

for (i in 1:20){
  col = paste0("C.",i)
  dLNom[,col] <- abs(sol[i,"Soln"]-dLNom[,col])
  dWNom[,col] <- abs(sol[i,"Soln"]-dWNom[,col])

  dLNomSign[,col] <- sol[i,"Soln"]-dLNomSign[,col]
  dWNomSign[,col] <- sol[i,"Soln"]-dWNomSign[,col]
}


dNom<-rbind(dLNom,dWNom)
dNomSign<-rbind(dLNomSign,dWNomSign)


# Create cells based on label / average accuracy is considered
dNom$pp <- (dNom$C.1 + dNom$C.2 + dNom$C.3 + dNom$C.4 + dNom$C.5)/5 
dNom$p <- (dNom$C.6 + dNom$C.7 + dNom$C.8 + dNom$C.9 + dNom$C.10)/5
dNom$m <- (dNom$C.11 + dNom$C.12 + dNom$C.13 + dNom$C.14 + dNom$C.15)/5 
dNom$mm <- (dNom$C.16 + dNom$C.17 + dNom$C.18 + dNom$C.19 + dNom$C.20)/5


# Create the cells based on signs / average accuracy is considered
dNom$ppSat <- (dNom$C.1 + dNom$C.2)/2
dNom$ppDen <- (dNom$C.4 + dNom$C.5)/2
dNom$pSat <- (dNom$C.6 + dNom$C.7)/2
dNom$pDen <- (dNom$C.9 + dNom$C.10)/2
dNom$mSat <- (dNom$C.11 + dNom$C.12)/2 
dNom$mDen <- (dNom$C.14 + dNom$C.15)/2
dNom$mmSat <- (dNom$C.16 + dNom$C.17)/2
dNom$mmDen <- (dNom$C.19 + dNom$C.20)/2


# Create the cells for the non-absolute data set / average accuracy is again considered
dNomSign$pp <- (dNomSign$C.1 + dNomSign$C.2 + dNomSign$C.3 + dNomSign$C.4 + dNomSign$C.5)/5 
dNomSign$p <- (dNomSign$C.6 + dNomSign$C.7 + dNomSign$C.8 + dNomSign$C.9 + dNomSign$C.10)/5
dNomSign$m <- (dNomSign$C.11 + dNomSign$C.12 + dNomSign$C.13 + dNomSign$C.14 + dNomSign$C.15)/5 
dNomSign$mm <- (dNomSign$C.16 + dNomSign$C.17 + dNomSign$C.18 + dNomSign$C.19 + dNomSign$C.20)/5

dNomSign$ppSat <- (dNomSign$C.1 + dNomSign$C.2)/2
dNomSign$ppDen <- (dNomSign$C.4 + dNomSign$C.5)/2
dNomSign$pSat <- (dNomSign$C.6 + dNomSign$C.7)/2
dNomSign$pDen <- (dNomSign$C.9 + dNomSign$C.10)/2
dNomSign$mSat <- (dNomSign$C.11 + dNomSign$C.12)/2 
dNomSign$mDen <- (dNomSign$C.14 + dNomSign$C.15)/2
dNomSign$mmSat <- (dNomSign$C.16 + dNomSign$C.17)/2
dNomSign$mmDen <- (dNomSign$C.19 + dNomSign$C.20)/2

# Special Case No origin
dNomNo <- dNomSign
dNomNo$pp <- dNomNo$C.3
dNomNo$p <- dNomNo$C.8
dNomNo$m <- dNomNo$C.13
dNomNo$mm <- dNomNo$C.18


#
#
# ABSOLUTE distance data sets
#

# Organized by labels
dAbsLab <- dNom[,c("RID","Group","pp","p","m","mm")]
dAbsLab$Group <- factor(dAbsLab$Group)
dAbsLab$RID <- factor(dAbsLab$RID)
dAbsLabL <- melt(dAbsLab,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")

dAbsLabL[dAbsLabL$Type %in% c("pp","p"),"Sign"] <- "Plus"
dAbsLabL[dAbsLabL$Type %in% c("m","mm"),"Sign"] <- "Minus"
dAbsLabL[dAbsLabL$Type %in% c("pp","mm"),"Intensity"] <- "Double"
dAbsLabL[dAbsLabL$Type %in% c("m","p"),"Intensity"] <- "Single"

dAbsLabL$Sign <- factor(dAbsLabL$Sign,levels = c("Plus","Minus"))
dAbsLabL$Intensity <- factor(dAbsLabL$Intensity,levels = c("Single","Double"))

# Organized by sign and satisfaction
dAbsSat <- dNom[,c("RID","Group","ppSat","ppDen","pSat","pDen","mSat","mDen","mmSat","mmDen")]
dAbsSatL <- melt(dAbsSat,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")

dAbsSatL[dAbsSatL$Type %in% c("ppSat","pSat","mSat","mmSat"),"Origin"] <- "Sat"
dAbsSatL[dAbsSatL$Type %in% c("ppDen","pDen","mDen","mmDen"),"Origin"] <- "Den"
dAbsSatL[dAbsSatL$Type %in% c("ppSat","pSat","ppDen","pDen"),"Sign"] <- "Plus"
dAbsSatL[dAbsSatL$Type %in% c("mSat","mmSat","mDen","mmDen"),"Sign"] <- "Minus"

dAbsSatL$Sign <- factor(dAbsSatL$Sign,levels = c("Plus","Minus"))
dAbsSatL$Origin <- factor(dAbsSatL$Origin,levels = c("Sat","Den"))


#
#
# RELATIVE distance data sets
#

# Organized by labels
dRelLab <- dNomSign[,c("RID","Group","pp","p","m","mm")]
dRelLab$Group <- factor(dRelLab$Group)
dRelLab$RID <- factor(dRelLab$RID)
dRelLabL <- melt(dRelLab,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")

dRelLabL[dRelLabL$Type %in% c("pp","p"),"Sign"] <- "Plus"
dRelLabL[dRelLabL$Type %in% c("m","mm"),"Sign"] <- "Minus"
dRelLabL[dRelLabL$Type %in% c("pp","mm"),"Intensity"] <- "Double"
dRelLabL[dRelLabL$Type %in% c("m","p"),"Intensity"] <- "Single"

dRelLabL$Sign <- factor(dRelLabL$Sign,levels = c("Plus","Minus"))
dRelLabL$Intensity <- factor(dRelLabL$Intensity,levels = c("Single","Double"))


# Special Case:  
dRelNo <- dNomNo[,c("RID","Group","pp","p","m","mm")]
dRelNo$Group <- factor(dRelNo$Group)
dRelNo$RID <- factor(dRelNo$RID)
dRelNoL <- melt(dRelNo,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")


# Organized by sign and satisfaction 
dRelSat <- dNomSign[,c("RID","Group","ppSat","ppDen","pSat","pDen","mSat","mDen","mmSat","mmDen")]
dRelSatL <- melt(dRelSat,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")

dRelSatL[dRelSatL$Type %in% c("ppSat","pSat","mSat","mmSat"),"Origin"] <- "Sat"
dRelSatL[dRelSatL$Type %in% c("ppDen","pDen","mDen","mmDen"),"Origin"] <- "Den"
dRelSatL[dRelSatL$Type %in% c("ppSat","pSat","ppDen","pDen"),"Sign"] <- "Plus"
dRelSatL[dRelSatL$Type %in% c("mSat","mmSat","mDen","mmDen"),"Sign"] <- "Minus"

dRelSatL$Sign <- factor(dRelSatL$Sign,levels = c("Plus","Minus"))
dRelSatL$Origin <- factor(dRelSatL$Origin,levels = c("Sat","Den"))

Relative Deviation from Normative (I)

To measure the distance between the participant responses and the normative ones according to the formal semantics (or simply the accuracy of the participants’ responses), we again coded both responses to the scale \([1,5]\) which we interpreted as interval. For each single response \(i\) on exercise \(j\) two distances are calculated: relative distance \(d_i = obs_{ij} - norm_j\) and absolute distance \(|d_i|\). When the former is positive participants overestimate satisfaction of the destination and vice-versa.

Boxplots

#Overal
brd1 <- ggplot(data = dRelLabL,aes(x=Group,y=Deviation,fill=Type)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)") + theme(text = element_text(size=18))

brd5 <- ggplot(data = dRelLabL,aes(x=Group,y=Deviation,fill=Intensity)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)")


brd6 <- ggplot(data = dRelLabL,aes(x=Sign,y=Deviation,fill=Intensity)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)")

brd1

In the top left figure pp,p,n,nn stand for “\(++\)”,“\(+\)”,“\(-\)”,“\(--\)” or make, help, hurt, break depending on the group considered. In the same figure it is clear that with positive labels responses satisfaction of the destination goal is overestimated and with negative is underestimated.

grid.arrange(brd5,brd6,ncol=2)

The top left figure seems to also show that when the origin goal is denied, the underestimation is more pronounced.

Is relative deviation zero?

Group x Sign x Intensity

The following are the p-values of one sample t-tests (as well as the non-parametric equivalent) that the distance from normative is zero (0) of each of the \(2\times 2\times2\) (visualization group,label sign, label intensity) cells, as well as the \(2\times 2 \times 2\) (visualization group, label sign, origin satisfaction) cell.

Relative distance is meaningful for this test. They are all independent comparisons.

testRow <- function (df,group,label){
  l <- list(paste0(group,".",label),
             t.test(df[df$Group == group,label], mu=0)$p.value,
             wilcox.test(df[df$Group == group,label], mu=0, exact=FALSE)$p.value,
             mean(df[df$Group == group,label]),
             t.test(df[df$Group == group,label], mu=0)$conf.int[1], 
             t.test(df[df$Group == group,label], mu=0)$conf.int[2]
            )
  return (l)
}

z<-data.frame(stringsAsFactors = FALSE)
z<-rbind(z,testRow(dNomSign,"Symbolic","pp"))
z[,1] <- as.character(z[,1])
z<-rbind(z,testRow(dNomSign,"Symbolic","p"))
z<-rbind(z,testRow(dNomSign,"Symbolic","m"))
z<-rbind(z,testRow(dNomSign,"Symbolic","mm"))
z<-rbind(z,testRow(dNomSign,"Textual","pp"))
z<-rbind(z,testRow(dNomSign,"Textual","p"))
z<-rbind(z,testRow(dNomSign,"Textual","m"))
z<-rbind(z,testRow(dNomSign,"Textual","mm"))


z<-rbind(z,testRow(dNomSign,"Symbolic","ppSat"))
z<-rbind(z,testRow(dNomSign,"Symbolic","pSat"))
z<-rbind(z,testRow(dNomSign,"Symbolic","mSat"))
z<-rbind(z,testRow(dNomSign,"Symbolic","mmSat"))
z<-rbind(z,testRow(dNomSign,"Symbolic","ppDen"))
z<-rbind(z,testRow(dNomSign,"Symbolic","pDen"))
z<-rbind(z,testRow(dNomSign,"Symbolic","mDen"))
z<-rbind(z,testRow(dNomSign,"Symbolic","mmDen"))

z<-rbind(z,testRow(dNomSign,"Textual","ppSat"))
z<-rbind(z,testRow(dNomSign,"Textual","pSat"))
z<-rbind(z,testRow(dNomSign,"Textual","mSat"))
z<-rbind(z,testRow(dNomSign,"Textual","mmSat"))
z<-rbind(z,testRow(dNomSign,"Textual","ppDen"))
z<-rbind(z,testRow(dNomSign,"Textual","pDen"))
z<-rbind(z,testRow(dNomSign,"Textual","mDen"))
z<-rbind(z,testRow(dNomSign,"Textual","mmDen"))


names(z)[1] = "Group"
names(z)[2] = "t-test"
names(z)[3] = "Wilcoxon Signed Rank"
names(z)[4] = "Mean Deviation"
names(z)[5] = "95 ConfInt - L"
names(z)[6] = "95 ConfInt - U"

z[z[,3]<0.05,"Sign. (* means p < 0.5)"] <- "* (non-zero)"
z[z[,3]>=0.05,"Sign. (* means p < 0.5)"] <- ""

kable(z[,c(1,4,7)],digits = 2)

Group	Mean Deviation	Sign. (* means p < 0.5)
Symbolic.pp	0.49	* (non-zero)
Symbolic.p	0.14
Symbolic.m	-0.31	* (non-zero)
Symbolic.mm	-0.38	* (non-zero)
Textual.pp	0.61	* (non-zero)
Textual.p	0.29	* (non-zero)
Textual.m	-0.20
Textual.mm	-0.49	* (non-zero)
Symbolic.ppSat	0.17	* (non-zero)
Symbolic.pSat	0.39	* (non-zero)
Symbolic.mSat	0.36
Symbolic.mmSat	0.72	* (non-zero)
Symbolic.ppDen	0.83	* (non-zero)
Symbolic.pDen	-0.14
Symbolic.mDen	-1.03	* (non-zero)
Symbolic.mmDen	-1.44	* (non-zero)
Textual.ppSat	-0.09
Textual.pSat	0.21
Textual.mSat	0.62
Textual.mmSat	0.82	* (non-zero)
Textual.ppDen	1.41	* (non-zero)
Textual.pDen	0.47
Textual.mDen	-1.00	* (non-zero)
Textual.mmDen	-1.88	* (non-zero)

Overestimation vs. Underestimation

We now investigate under what circumstances participants overestimate and underestimate satisfaction. In the table below the average overestimation (positive value) or underestimation (negative value) per visualization style, contribution label and satisfaction origin is displayed.

dRelSatL2 <- dRelSatL

dRelSatL2[dRelSatL2$Type %in% c("ppSat","ppDen"),"Label"] <- "Make (++)"
dRelSatL2[dRelSatL2$Type %in% c("pSat","pDen"),"Label"] <- "Help (+)"
dRelSatL2[dRelSatL2$Type %in% c("mSat","mDen"),"Label"] <- "Hurt (-)"
dRelSatL2[dRelSatL2$Type %in% c("mmSat","mmDen"),"Label"] <- "Break (--)"

dRelSatL2$Label <- factor(dRelSatL2$Label,levels = c("Make (++)","Help (+)","Hurt (-)","Break (--)"))

dRelSatLCross <- dRelSatL2[,c("Origin","Label","Deviation","Group")]


tab1 <- xtabs(Deviation ~ Group+Origin+Label,
             aggregate(Deviation ~ Group+Origin+Label,
                       dRelSatLCross,
                       mean)
             )

ftable(tab1)

##                 Label   Make (++)    Help (+)    Hurt (-)  Break (--)
## Group    Origin                                                      
## Symbolic Sat           0.16666667  0.38888889  0.36111111  0.72222222
##          Den           0.83333333 -0.13888889 -1.02777778 -1.44444444
## Textual  Sat          -0.08823529  0.20588235  0.61764706  0.82352941
##          Den           1.41176471  0.47058824 -1.00000000 -1.88235294

#xtableFtable(ftable(tab1))

There are some differences between the two visualizations in terms of overestimation and underestimation. In addition, extreme labels (\(++\)) and (\(--\)) may naturally feature greater error. Cases in which the average error is consistent and substantial (\(>0.7\)) are:

1. hurt and break labels with denied origin goals, where satisfaction is underestimated.

2. break labels with satisfied origin goals, where satisfaction is overestimated. As above participants do not seem to perceive the satisfaction inversion of negative labels.

3. make links with denied origin goals, where satisfaction is overestimated.

Looking at the descriptive images above, it becomes clear that many participants do not seem to perceive the satisfaction inversion of negative labels (cases (a) and (b)). In addition they do not seem to accept that even a strong makes relationship can result in a fully denied destination goal.

If we focus exclusively on cases in which the satisfaction of the origin is labelled as “no-information” (N) the following table describes the average deviation from normative.

dRelNoL[dRelNoL$Type == "pp","Label"] <- "Make (++)" 
dRelNoL[dRelNoL$Type == "p","Label"] <- "Help (+)" 
dRelNoL[dRelNoL$Type == "m","Label"] <- "Hurt (-)" 
dRelNoL[dRelNoL$Type == "mm","Label"] <- "Break (--)" 


dRelNoL$Label <- factor(dRelNoL$Label, levels = c("Make (++)","Help (+)","Hurt (-)","Break (--)"))

dRelNoLCross <- dRelNoL[,c("Label","Deviation","Group")]

tab2 <- xtabs(Deviation ~ Group+Label,
             aggregate(Deviation ~ Group+Label,
                       dRelNoLCross,
                       mean)
             )
ftable(tab2)

##          Label  Make (++)   Help (+)   Hurt (-) Break (--)
## Group                                                     
## Symbolic        0.4444444  0.2222222 -0.2222222 -0.4444444
## Textual         0.4117647  0.1176471 -0.2352941 -0.3529412

#xtable(tab2)

It seems that users perceive contributions as generators of satisfaction than just propagators thereof. Positive contributions result in some satisfaction, and negative contributions result to some denial irrespective of the satisfaction of the origin. This mental model of labels conflicts with the links’ ability to completely inverse satisfaction, which users also had some trouble to comprehend.

Assumptions

Homogeity - Box’s test

boxM(dRelLab[,-c(1,2)],dRelLab[,2])

## 
##  Box's M-test for Homogeneity of Covariance Matrices
## 
## data:  dRelLab[, -c(1, 2)]
## Chi-Sq (approx.) = 14.706, df = 10, p-value = 0.1431

Box’s test does not meet the significance test; we can assume homogeneity of covariance matrices.

Normality Tests

mshapiro.test(t(dRelLab[dRelLab$Group=="Symbolic",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.79087, p-value = 0.001132

mshapiro.test(t(dRelLab[dRelLab$Group=="Textual",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.59411, p-value = 9.134e-06

QQ-Plots

par(mfrow=c(4,2))

qqnorm(dRelLab[dRelLab$Group=="Textual",c("pp")], main = "Textual - pp")
qqnorm(dRelLab[dRelLab$Group=="Symbolic",c("pp")], main = "LSymbolic - pp")
qqnorm(dRelLab[dRelLab$Group=="Textual",c("p")], main = "Textual - p")
qqnorm(dRelLab[dRelLab$Group=="Symbolic",c("p")], main = "Symbolic - p")
qqnorm(dRelLab[dRelLab$Group=="Textual",c("m")], main = "Textual - m")
qqnorm(dRelLab[dRelLab$Group=="Symbolic",c("m")], main = "Symbolic - m")
qqnorm(dRelLab[dRelLab$Group=="Textual",c("mm")], main = "Textual - mm")
qqnorm(dRelLab[dRelLab$Group=="Symbolic",c("mm")], main = "Symbolic - mm")

We do seem to have issues with normality and the small sample size does not entirely allow us to deal with it by appealing to robustness arguments.

MANOVA

#
# Multi-variate test with repeated measures.
# Please consult J.Fox and  S. Weisberg's 
# "Multivariate Linear Models in R" 
#
mod <- lm(cbind(pp,p,m,mm) ~ Group,data = dRelLab)
iData <- data.frame(Sig = c("Plus","Plus","Minus","Minus"),Intensity = c("Double","Single","Single","Double"))
iData$Sig <- factor(iData$Sig) 
iData$Intensity <- factor(iData$Intensity)
(mod.aov <- Anova(mod, idata = iData,idesign = ~Sig*Intensity, type=3))

## 
## Type III Repeated Measures MANOVA Tests: Pillai test statistic
##                     Df test stat approx F num Df den Df   Pr(>F)   
## (Intercept)          1  0.001781   0.0589      1     33 0.809793   
## Group                1  0.019667   0.6620      1     33 0.421674   
## Sig                  1  0.198117   8.1531      1     33 0.007378 **
## Group:Sig            1  0.005268   0.1748      1     33 0.678606   
## Intensity            1  0.178278   7.1596      1     33 0.011516 * 
## Group:Intensity      1  0.081121   2.9133      1     33 0.097245 . 
## Sig:Intensity        1  0.097017   3.5455      1     33 0.068546 . 
## Group:Sig:Intensity  1  0.012279   0.4102      1     33 0.526271   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Sign Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Sig")

## 
##  Response transformation matrix:
##    Sig1
## pp   -1
## p    -1
## m     1
## mm    1
## 
## Sum of squares and products for the hypothesis:
##          Sig1
## Sig1 31.46889
## 
## Sum of squares and products for error:
##          Sig1
## Sig1 127.3711
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df    Pr(>F)   
## Pillai            1 0.1981169 8.153131      1     33 0.0073777 **
## Wilks             1 0.8018831 8.153131      1     33 0.0073777 **
## Hotelling-Lawley  1 0.2470646 8.153131      1     33 0.0073777 **
## Roy               1 0.2470646 8.153131      1     33 0.0073777 **
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

We have significant effect on Sign \(F(1,33)=8.1531, p=0.007378\)

Intensity Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Intensity")

## 
##  Response transformation matrix:
##    Intensity1
## pp          1
## p          -1
## m          -1
## mm          1
## 
## Sum of squares and products for the hypothesis:
##            Intensity1
## Intensity1   1.388889
## 
## Sum of squares and products for error:
##            Intensity1
## Intensity1   6.401699
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df   Pr(>F)  
## Pillai            1 0.1782778 7.159557      1     33 0.011516 *
## Wilks             1 0.8217222 7.159557      1     33 0.011516 *
## Hotelling-Lawley  1 0.2169563 7.159557      1     33 0.011516 *
## Roy               1 0.2169563 7.159557      1     33 0.011516 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

We have significant effect on Intensity \(F(1,33)=7.1596, p=0.011516\)

Relative Deviation from Normative (II)

In these tests, we utilize a different aggregation of the data, in which the satisfaction of the origin is also considered as one of the factors. In this data set the data points coming from models in which satisfaction of origin is “No Information” (N) are excluded.

BoxPlots

brd2 <- ggplot(data = dRelSatL,aes(x=Sign,y=Deviation,fill=Origin)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)")

brd3 <- ggplot(data = dRelSatL,aes(x=Group,y=Deviation,fill=Sign)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)")

brd4 <- ggplot(data = dRelSatL,aes(x=Group,y=Deviation,fill=Origin)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (rel)")

grid.arrange(brd2,brd3,brd4,ncol=3)

Assumptions

Homogeity - Box’s test

dRelSatPr <- dRelSat[-15,]
#table(dRelSatPr$Group)
boxM(dRelSatPr[,-c(1,2)],dRelSatPr[,2])

## 
##  Box's M-test for Homogeneity of Covariance Matrices
## 
## data:  dRelSatPr[, -c(1, 2)]
## Chi-Sq (approx.) = 64.497, df = 36, p-value = 0.002446

Given the number of DVs here it is not suprizing that Box’s M test fails. Following Tabachnick and Fidell, we randomly remove one of the cases in order to bring all cells to equal size. In this case and given we do not fail for \(p<0.001\) we can proceed with the analysis.

Normality Tests

mshapiro.test(t(dRelSatPr[dRelSatPr$Group=="Symbolic",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.58052, p-value = 6.784e-06

mshapiro.test(t(dRelSatPr[dRelSatPr$Group=="Textual",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.51064, p-value = 1.597e-06

par(mfrow=c(4,2))

qqnorm(dRelSatPr[dRelSatPr$Group=="Textual",c("pp")], main = "Textual - pp")
qqnorm(dRelSatPr[dRelSatPr$Group=="Symbolic",c("pp")], main = "LSymbolic - pp")
qqnorm(dRelSatPr[dRelSatPr$Group=="Textual",c("p")], main = "Textual - p")
qqnorm(dRelSatPr[dRelSatPr$Group=="Symbolic",c("p")], main = "Symbolic - p")
qqnorm(dRelSatPr[dRelSatPr$Group=="Textual",c("m")], main = "Textual - m")
qqnorm(dRelSatPr[dRelSatPr$Group=="Symbolic",c("m")], main = "Symbolic - m")
qqnorm(dRelSatPr[dRelSatPr$Group=="Textual",c("mm")], main = "Textual - mm")
qqnorm(dRelSatPr[dRelSatPr$Group=="Symbolic",c("mm")], main = "Symbolic - mm")

(see commentary above on normality)

MANOVA

#
# Multi-variate test with repeated measures.
# Please consult J.Fox and  S. Weisberg's 
# "Multivariate Linear Models in R" 
#


mod <- lm(cbind(ppSat,ppDen,pSat,pDen,mSat,mDen,mmSat,mmDen) ~ Group,data = dRelSatPr)
iData <- data.frame(Sig = c(rep("Plus",4),rep("Minus",4)),Origin = rep(c("Sat","Den"),4))
iData$Sig <- factor(iData$Sig)
iData$Origin <- factor(iData$Origin)
(mod.aov <- Anova(mod, idata = iData,idesign = ~Sig*Origin, type=3))

## 
## Type III Repeated Measures MANOVA Tests: Pillai test statistic
##                  Df test stat approx F num Df den Df   Pr(>F)   
## (Intercept)       1  0.001860   0.0596      1     32 0.808654   
## Group             1  0.021013   0.6868      1     32 0.413381   
## Sig               1  0.195266   7.7647      1     32 0.008884 **
## Group:Sig         1  0.006464   0.2082      1     32 0.651259   
## Origin            1  0.275118  12.1451      1     32 0.001450 **
## Group:Origin      1  0.015996   0.5202      1     32 0.475996   
## Sig:Origin        1  0.207551   8.3811      1     32 0.006778 **
## Group:Sig:Origin  1  0.033058   1.0940      1     32 0.303418   
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Sign Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Origin, iterms="Sig")

## 
##  Response transformation matrix:
##       Sig1
## ppSat   -1
## ppDen   -1
## pSat    -1
## pDen    -1
## mSat     1
## mDen     1
## mmSat    1
## mmDen    1
## 
## Sum of squares and products for the hypothesis:
##          Sig1
## Sig1 132.7206
## 
## Sum of squares and products for error:
##          Sig1
## Sig1 546.9706
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df   Pr(>F)   
## Pillai            1 0.1952660 7.764693      1     32 0.008884 **
## Wilks             1 0.8047340 7.764693      1     32 0.008884 **
## Hotelling-Lawley  1 0.2426467 7.764693      1     32 0.008884 **
## Roy               1 0.2426467 7.764693      1     32 0.008884 **
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

There is a main effect on Sign \(F(1,32)=7.7647, p=0.008884\).

Origin Satisfaction Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Origin, iterms="Origin")

## 
##  Response transformation matrix:
##       Origin1
## ppSat      -1
## ppDen       1
## pSat       -1
## pDen        1
## mSat       -1
## mDen        1
## mmSat      -1
## mmDen       1
## 
## Sum of squares and products for the hypothesis:
##          Origin1
## Origin1 222.4853
## 
## Sum of squares and products for error:
##          Origin1
## Origin1 586.2059
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df    Pr(>F)   
## Pillai            1 0.2751177  12.1451      1     32 0.0014504 **
## Wilks             1 0.7248823  12.1451      1     32 0.0014504 **
## Hotelling-Lawley  1 0.3795344  12.1451      1     32 0.0014504 **
## Roy               1 0.3795344  12.1451      1     32 0.0014504 **
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

There is a main effect on Satisfactin of Origin goal \(F(1,32) = 12.1451, p=0.00145\).

Interaction

dRelSatPrL <- melt(dRelSatPr,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Deviation")

dRelSatPrL[dRelSatPrL$Type %in% c("ppSat","pSat","mSat","mmSat"),"Origin"] <- "Sat"
dRelSatPrL[dRelSatPrL$Type %in% c("ppDen","pDen","mDen","mmDen"),"Origin"] <- "Den"
dRelSatPrL[dRelSatPrL$Type %in% c("ppSat","pSat","ppDen","pDen"),"Sign"] <- "Plus"
dRelSatPrL[dRelSatPrL$Type %in% c("mSat","mmSat","mDen","mmDen"),"Sign"] <- "Minus"

dRelSatPrL$Sign <- factor(dRelSatPrL$Sign)
dRelSatPrL$Origin <- factor(dRelSatPrL$Origin)

p1 <- ggplot(dRelSatPrL, aes(x = Origin, y = Deviation,  color = Sign)) +  stat_summary(fun.y = mean, geom = "line", aes(group=Sign), linetype = "solid", size = 1) + stat_summary(fun.data = mean_cl_boot, geom = "errorbar", width = 0.1, size = 1) + ylab("Deviation from Normative (rel)") + theme(legend.position = "top")

p2 <- ggplot(dRelSatPrL, aes(x = Sign, y = Deviation,  color = Origin)) +  stat_summary(fun.y = mean, geom = "line", aes(group=Origin), linetype = "solid", size = 1) + stat_summary(fun.data = mean_cl_boot, geom = "errorbar", width = 0.1, size = 1) + ylab("Deviation from Normative (rel)") + theme(legend.position = "top")

p1

The above interaction is fairly intuitive: for negative contributions a denial of the origin leads to underestimation of the destination satisfaction (participants do not perceive satisfaction inversion). Positive contributions appear to be perceived as “blocking” the denial of the origin goal, hence the slight overestimation in such configurations.

Absolute Deviation from Normative (I)

We now turn to deviations based on absolute distance.

BoxPlots

bad1 <- ggplot(data = dAbsLabL,aes(x=Group,y=Deviation,fill=Type)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)")

bad2 <- ggplot(data = dAbsLabL,aes(x=Group,y=Deviation,fill=Sign)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)")

bad3 <- ggplot(data = dAbsLabL,aes(x=Group,y=Deviation,fill=Intensity)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)")

bad4 <- ggplot(data = dAbsLabL,aes(x=Sign,y=Deviation,fill=Intensity)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)")

bad1

grid.arrange(bad2,bad3,bad4,ncol=3)

Assumptions

Homogeity - Box’s test

boxM(dAbsLab[,-c(1,2)],dAbsLab[,2])

## 
##  Box's M-test for Homogeneity of Covariance Matrices
## 
## data:  dAbsLab[, -c(1, 2)]
## Chi-Sq (approx.) = 13.059, df = 10, p-value = 0.2204

Box’s test does not meet the significance test; we can assume homogeneity of covariance matrices.

Normality Tests

mshapiro.test(t(dAbsLab[dAbsLab$Group=="Symbolic",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.88285, p-value = 0.02916

mshapiro.test(t(dAbsLab[dAbsLab$Group=="Textual",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.84106, p-value = 0.007859

QQ-Plots

par(mfrow=c(4,2))

qqnorm(dAbsLab[dAbsLab$Group=="Textual",c("pp")], main = "Textual - pp")
qqnorm(dAbsLab[dAbsLab$Group=="Symbolic",c("pp")], main = "LSymbolic - pp")
qqnorm(dAbsLab[dAbsLab$Group=="Textual",c("p")], main = "Textual - p")
qqnorm(dAbsLab[dAbsLab$Group=="Symbolic",c("p")], main = "Symbolic - p")
qqnorm(dAbsLab[dAbsLab$Group=="Textual",c("m")], main = "Textual - m")
qqnorm(dAbsLab[dAbsLab$Group=="Symbolic",c("m")], main = "Symbolic - m")
qqnorm(dAbsLab[dAbsLab$Group=="Textual",c("mm")], main = "Textual - mm")
qqnorm(dAbsLab[dAbsLab$Group=="Symbolic",c("mm")], main = "Symbolic - mm")

We have deviation from normality which furthermore cannot be cured with transformations. Judging from the “quantized” appearance of the graphs we can suspect that the culprit is the discrete scale [0..4] for measuring distance combined probably with the small sample size. The tests that follow are based on the assumption of their robustness to such deviations suggested in Tabachnick and Fidel and that for “large” \(N\) the problem is less critical.

Hypotheses Testing

#
# Multi-variate test with repeated measures.
# Please consult J.Fox and  S. Weisberg's 
# "Multivariate Linear Models in R" 
#
mod <- lm(cbind(pp,p,m,mm) ~ Group,data = dAbsLab)
iData <- data.frame(Sig = c("Plus","Plus","Minus","Minus"),Intensity = c("Double","Single","Single","Double"))
iData$Sig <- factor(iData$Sig) 
iData$Intensity <- factor(iData$Intensity)
(mod.aov <- Anova(mod, idata = iData,idesign = ~Sig*Intensity, type=3))

## 
## Type III Repeated Measures MANOVA Tests: Pillai test statistic
##                     Df test stat approx F num Df den Df    Pr(>F)    
## (Intercept)          1   0.53737   38.331      1     33 5.502e-07 ***
## Group                1   0.02478    0.839      1     33   0.36646    
## Sig                  1   0.17961    7.225      1     33   0.01118 *  
## Group:Sig            1   0.00335    0.111      1     33   0.74111    
## Intensity            1   0.01794    0.603      1     33   0.44298    
## Group:Intensity      1   0.00868    0.289      1     33   0.59458    
## Sig:Intensity        1   0.05417    1.890      1     33   0.17848    
## Group:Sig:Intensity  1   0.02010    0.677      1     33   0.41651    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

There are significant main effects in Sign \(F(1,33)=7.225, p<0.05\) and pretty much this is all there is.

Sign Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Sig")

## 
##  Response transformation matrix:
##    Sig1
## pp   -1
## p    -1
## m     1
## mm    1
## 
## Sum of squares and products for the hypothesis:
##          Sig1
## Sig1 16.05556
## 
## Sum of squares and products for error:
##          Sig1
## Sig1 73.33503
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df   Pr(>F)  
## Pillai            1 0.1796113 7.224833      1     33 0.011179 *
## Wilks             1 0.8203887 7.224833      1     33 0.011179 *
## Hotelling-Lawley  1 0.2189343 7.224833      1     33 0.011179 *
## Roy               1 0.2189343 7.224833      1     33 0.011179 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Intensity Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Intensity")

## 
##  Response transformation matrix:
##    Intensity1
## pp          1
## p          -1
## m          -1
## mm          1
## 
## Sum of squares and products for the hypothesis:
##            Intensity1
## Intensity1  0.6422222
## 
## Sum of squares and products for error:
##            Intensity1
## Intensity1   35.14837
## 
## Multivariate Tests: 
##                  Df test stat  approx F num Df den Df  Pr(>F)
## Pillai            1 0.0179439 0.6029678      1     33 0.44298
## Wilks             1 0.9820561 0.6029678      1     33 0.44298
## Hotelling-Lawley  1 0.0182718 0.6029678      1     33 0.44298
## Roy               1 0.0182718 0.6029678      1     33 0.44298

There is no Intensity effect.

Absolute Deviation from Normative - (II)

To consider satisfaction of origin as one of the factors, a different data set is prepared, in which the N (no information) satisfaction value is eliminated.

bad5 <- ggplot(data = dAbsSatL,aes(x=Sign,y=Deviation,fill=Origin)) + geom_boxplot() + scale_fill_discrete(name = "Origin Sat. Status") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)") + scale_fill_manual(values = c("#5599FF","#FF5555"))

## Scale for 'fill' is already present. Adding another scale for 'fill',
## which will replace the existing scale.

bad7 <- ggplot(data = dAbsSatL,aes(x=Group,y=Deviation,fill=Origin)) + geom_boxplot() + scale_fill_discrete(name = "Origin Sat. Status") + theme(legend.position = "top") + ylab("Deviation from Normative (abs)") + scale_fill_manual(values = c("#5599FF","#FF5555"))

## Scale for 'fill' is already present. Adding another scale for 'fill',
## which will replace the existing scale.

grid.arrange(bad5,bad7,ncol=2)

Assumptions

Homogeity - Box’s test

boxM(dAbsSat[,-c(1,2)],dAbsSat[,2])

## 
##  Box's M-test for Homogeneity of Covariance Matrices
## 
## data:  dAbsSat[, -c(1, 2)]
## Chi-Sq (approx.) = 59.233, df = 36, p-value = 0.008688

#Randomly remove a case from Symbolic to equate sample sizes / found (3)
dAbsSatPr <- dAbsSat[-3,]

boxM(dAbsSatPr[,-c(1,2)],dAbsSatPr[,2])

## 
##  Box's M-test for Homogeneity of Covariance Matrices
## 
## data:  dAbsSatPr[, -c(1, 2)]
## Chi-Sq (approx.) = 55.805, df = 36, p-value = 0.01868

Given the number of DVs here it is not suprizing that Box’s M test fails. Following Tabachnick and Fidell, we randomly remove one of the cases in order to bring all cells to equal size. In this case and given we do not fail for \(p<0.001\) we can proceed with the analysis.

Normality Tests

mshapiro.test(t(dAbsSatPr[dAbsSatPr$Group=="Symbolic",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.58437, p-value = 7.377e-06

mshapiro.test(t(dAbsSatPr[dAbsSatPr$Group=="Textual",-c(1,2)]))

## 
##  Shapiro-Wilk normality test
## 
## data:  Z
## W = 0.53423, p-value = 2.564e-06

See above for discussion on normality.

MANOVA

#
# Multi-variate test with repeated measures.
# Please consult J.Fox and  S. Weisberg's 
# "Multivariate Linear Models in R" 



mod <- lm(cbind(ppSat,ppDen,pSat,pDen,mSat,mDen,mmSat,mmDen) ~ Group,data = dAbsSatPr)
iData <- data.frame(Sig = c(rep("Plus",4),rep("Minus",4)),Origin = rep(c("Sat","Den"),4))
iData$Sig <- factor(iData$Sig) 
iData$Origin <- factor(iData$Origin)
(mod.aov <- Anova(mod, idata = iData,idesign = ~Sig*Origin, type=3))

## 
## Type III Repeated Measures MANOVA Tests: Pillai test statistic
##                  Df test stat approx F num Df den Df    Pr(>F)    
## (Intercept)       1   0.55736   40.293      1     32 3.982e-07 ***
## Group             1   0.01968    0.642      1     32  0.428746    
## Sig               1   0.22793    9.447      1     32  0.004301 ** 
## Group:Sig         1   0.01126    0.364      1     32  0.550297    
## Origin            1   0.17249    6.670      1     32  0.014582 *  
## Group:Origin      1   0.00088    0.028      1     32  0.867480    
## Sig:Origin        1   0.00408    0.131      1     32  0.719534    
## Group:Sig:Origin  1   0.00051    0.016      1     32  0.898887    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

There is a significant effect with Sign \(F(1,32) = 9.447, p<0.01\) and the satisfaction value of the origin \(F(1,32) = 6.67, p<0.05\). As we saw in the graphs, both negative sings and denial values cause more deviation from the formal semantics.

Sign Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Origin, iterms="Sig")

## 
##  Response transformation matrix:
##       Sig1
## ppSat   -1
## ppDen   -1
## pSat    -1
## pDen    -1
## mSat     1
## mDen     1
## mmSat    1
## mmDen    1
## 
## Sum of squares and products for the hypothesis:
##          Sig1
## Sig1 119.1176
## 
## Sum of squares and products for error:
##       Sig1
## Sig1 403.5
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df    Pr(>F)   
## Pillai            1  0.227925 9.446753      1     32 0.0043013 **
## Wilks             1  0.772075 9.446753      1     32 0.0043013 **
## Hotelling-Lawley  1  0.295211 9.446753      1     32 0.0043013 **
## Roy               1  0.295211 9.446753      1     32 0.0043013 **
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Satisfaction Origin Effect

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Origin, iterms="Origin")

## 
##  Response transformation matrix:
##       Origin1
## ppSat      -1
## ppDen       1
## pSat       -1
## pDen        1
## mSat       -1
## mDen        1
## mmSat      -1
## mmDen       1
## 
## Sum of squares and products for the hypothesis:
##          Origin1
## Origin1 84.94118
## 
## Sum of squares and products for error:
##         Origin1
## Origin1   407.5
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df   Pr(>F)  
## Pillai            1 0.1724900 6.670227      1     32 0.014582 *
## Wilks             1 0.8275100 6.670227      1     32 0.014582 *
## Hotelling-Lawley  1 0.2084446 6.670227      1     32 0.014582 *
## Roy               1 0.2084446 6.670227      1     32 0.014582 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Confidence Analysis

To analyze response confidence and what affects it we first map the Likert-type scale to the scale {-2,-1,0,+1,+2}, and then sum up the correspondingly coded values per type of contribution link for each of the groups. We then proceed with the analysis as above. Neither visually nor through statistical tests do we find any main effect with respect to type of label to the response confidence of the participants.

The Data: BoxPlot

dLNom <- dLBU
dLNom$Group <- "Symbolic"
dLNom$RID<- paste0("L",dLNom$Response.ID)

dWNom <- dWBU
dWNom$Group <- "Textual"
dWNom$RID<- paste0("W",dWNom$Response.ID)


dNom<-rbind(dLNom,dWNom)

# Create cells -  use average
dNom$pp <- (dNom$C.1.c + dNom$C.2.c + dNom$C.3.c + dNom$C.4.c + dNom$C.5.c)/5 
dNom$p <- (dNom$C.6.c + dNom$C.7.c + dNom$C.8.c + dNom$C.9.c + dNom$C10.c)/5
dNom$m <- (dNom$C.11.c + dNom$C.12.c + dNom$C.13.c + dNom$C.14.c + dNom$C.15.c)/5 
dNom$mm <- (dNom$C.16.c + dNom$C.17.c + dNom$C.18.c + dNom$C.19.c + dNom$C.20.c)/5

dNom <- dNom[,c("RID","Group","pp","p","m","mm")]
dNomL <- melt(dNom,id.vars = c("RID","Group"),variable.name = "Type", value.name = "Confidence")
ggplot(data = dNomL,aes(x=Group,y=Confidence,fill=Type)) + geom_boxplot() + scale_fill_discrete(name = "Contr. Type")

Hypotheses Testing

dNom$Group <- factor(dNom$Group)
dNom$RID <- factor(dNom$RID)

mod <- lm(cbind(pp,p,m,mm) ~ Group,data = dNom)
iData <- data.frame(Sig = c("Plus","Plus","Minus","Minus"),Intensity = c("Double","Single","Single","Double"))
iData$Sig <- factor(iData$Sig) 
iData$Intensity <- factor(iData$Intensity) 
(mod.aov <- Anova(mod, idata = iData,idesign = ~Sig*Intensity, type=3))

## 
## Type III Repeated Measures MANOVA Tests: Pillai test statistic
##                     Df test stat approx F num Df den Df   Pr(>F)    
## (Intercept)          1   0.52017   35.774      1     33 1.02e-06 ***
## Group                1   0.00517    0.172      1     33  0.68142    
## Sig                  1   0.06982    2.477      1     33  0.12505    
## Group:Sig            1   0.00898    0.299      1     33  0.58823    
## Intensity            1   0.03388    1.157      1     33  0.28984    
## Group:Intensity      1   0.04030    1.386      1     33  0.24754    
## Sig:Intensity        1   0.09285    3.378      1     33  0.07510 .  
## Group:Sig:Intensity  1   0.10918    4.045      1     33  0.05254 .  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Sig")

## 
##  Response transformation matrix:
##    Sig1
## pp   -1
## p    -1
## m     1
## mm    1
## 
## Sum of squares and products for the hypothesis:
##           Sig1
## Sig1 0.8022222
## 
## Sum of squares and products for error:
##          Sig1
## Sig1 10.68719
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df  Pr(>F)
## Pillai            1 0.0698227 2.477109      1     33 0.12505
## Wilks             1 0.9301773 2.477109      1     33 0.12505
## Hotelling-Lawley  1 0.0750639 2.477109      1     33 0.12505
## Roy               1 0.0750639 2.477109      1     33 0.12505

linearHypothesis(mod, "(Intercept) = 0", idata=iData, idesign=~Sig*Intensity, iterms="Intensity")

## 
##  Response transformation matrix:
##    Intensity1
## pp          1
## p          -1
## m          -1
## mm          1
## 
## Sum of squares and products for the hypothesis:
##            Intensity1
## Intensity1  0.3755556
## 
## Sum of squares and products for error:
##            Intensity1
## Intensity1   10.70915
## 
## Multivariate Tests: 
##                  Df test stat approx F num Df den Df  Pr(>F)
## Pillai            1 0.0338805 1.157266      1     33 0.28984
## Wilks             1 0.9661195 1.157266      1     33 0.28984
## Hotelling-Lawley  1 0.0350687 1.157266      1     33 0.28984
## Roy               1 0.0350687 1.157266      1     33 0.28984

grid.arrange(asib1,asob2,ncol = 1)
brd1