Soil Protists in Three Neotropical Rainforests are Hyperdiverse and Dominated by Parasites

# kl
cd ${HOME}/neotropical_diversity/data/

# Assemble
PEAR="${HOME}/bin/PEAR/src/pear"
GUESS_ENCODING="${HOME}/src/guess-encoding.py"
THREADS=8

find . -name "*_1_1.fastq.bz2" | \
while read R1 ; do
    R2=$(sed -e 's/_1_1/_1_2/' <<< ${R1})
    bunzip2 -k ${R1} ${R2}
    ENCODING=$(awk 'NR % 4 == 0' ${R1/.bz2/} | python ${GUESS_ENCODING} 2> /dev/null)
    echo -e "${R1}\t${ENCODING}"
    ${PEAR} -b ${ENCODING} -j ${THREADS} -f ${R1/.bz2/} -r ${R2/.bz2/} -o ${R1/_1.fastq.bz2/}
    rm -f ${R1/_1.fastq.bz2/}{.discarded,.unassembled.{forward,reverse}}.fastq ${R1/.bz2/} ${R2/.bz2/}
done

exit 0

kl
cd ${HOME}/neotropical_diversity/data/2013-12-30/
for f in *.assembled.fastq ; do
    bash ../../src/clean_fastq_files.sh ${f}
done

#! /bin/bash

module load python/latest-2.7
export LC_ALL=C

INPUT="${1}"
CUTADAPT="/home/mahe/.local/bin/cutadapt"
VSEARCH="${HOME}/bin/vsearch/bin/vsearch-1.1.1-osx-x86_64"
HASHING="${HOME}/src/hashing.py"

PRIMER_F="CCAGCASCYGCGGTAATTCC"
PRIMER_R="TYRATCAAGAACGAAAGT"
ANTI_PRIMER_F="GGAATTACCGCRGSTGCTGG"
ANTI_PRIMER_R="ACTTTCGTTCTTGATYRA"

TMP_FORWARD=$(mktemp)
TMP_ANTI_FORWARD=$(mktemp)
TMP_REVERSE=$(mktemp)
TMP_ANTI_REVERSE=$(mktemp)
TMP_FASTA=$(mktemp)
TMP_FASTA_DEREPLICATED=$(mktemp)
FINAL_FASTA=${INPUT/.fastq/.fasta}
LOG=${INPUT/.fastq/.log}

# Get reads containing forward primer
${CUTADAPT} --discard-untrimmed \
    --format=fastq \
    -g PRIMER_F=${PRIMER_F} \
    -o ${TMP_FORWARD} ${INPUT} > ${LOG}

# Get reads containing reverse primer
${CUTADAPT} --discard-untrimmed \
    --format=fastq \
    -a PRIMER_R=${PRIMER_R} \
    -o ${TMP_REVERSE} ${TMP_FORWARD} >> ${LOG}

# Get reads containing anti-reverse primer (in 5' position)
${CUTADAPT} --discard-untrimmed \
    --format=fastq \
    -g ANTI_PRIMER_R=${ANTI_PRIMER_R} \
    -o ${TMP_ANTI_FORWARD} ${INPUT} >> ${LOG}

# Get reads containing anti-forward primer (in 3' position)
${CUTADAPT} --discard-untrimmed \
    --format=fastq \
    -a ANTI_PRIMER_F=${ANTI_PRIMER_F} \
    -o ${TMP_ANTI_REVERSE} ${TMP_ANTI_FORWARD} >> ${LOG}

# Convert fastq to fasta (reverse-complement the second file)
(awk '(NR - 2) % 4 == 0' ${TMP_REVERSE}
 awk '(NR - 2) % 4 == 0' ${TMP_ANTI_REVERSE} | \
     tr "acgturykmbdhvACGTURYKMBDHV" "tgcaayrmkvhdbTGCAAYRMKVHDB" | rev) | \
     grep -v [^ACGTacgt] | awk '{printf ">a%d\n%s\n", NR, $1}' > ${TMP_FASTA}

rm -f ${TMP_FORWARD} ${TMP_ANTI_FORWARD} ${TMP_REVERSE} ${TMP_ANTI_REVERSE}

# Dereplicate (vsearch)
"${VSEARCH}" --threads 1 \
    --derep_fulllength ${TMP_FASTA} \
    --sizeout \
    --fasta_width 0 \
    --output ${TMP_FASTA_DEREPLICATED}

# Compute hash values
python ${HASHING} ${TMP_FASTA_DEREPLICATED} > ${FINAL_FASTA}

# Get some basic statistics
awk -F "=" 'BEGIN {OFS = "\t"}
            /^>/ {c += 1 ; s += $2}
            END {
                printf "\n%s\n%s\t%d\n%s\t%d\n", "Basic stats:", "uniques", c, "reads", s
            }' ${FINAL_FASTA} >> ${LOG}

rm -f ${TMP_FASTA} ${TMP_FASTA_DEREPLICATED}

exit 0

kl
cd ${HOME}/neotropical_diversity/data/2013-12-30/
for TARGET in *_1_1.fastq.bz2 ; do
    ASSEMBLED=$(wc -l < ${TARGET/_1.fastq.bz2/.assembled.fastq})
    RAW=$(bzcat ${TARGET} | sed '/^$/d' | wc -l)
    CLEAN_READS=$(tail -n 1 ${TARGET/_1.fastq.bz2/.assembled.log} | cut -f 2)
    awk -v after=$ASSEMBLED -v before=$RAW -v file=$TARGET -v clean_reads=$CLEAN_READS 'BEGIN {printf "| %s | %s | %s | %.1f | %s | %.1f |\n", file, before / 4, after / 4, 100 * after / before, clean_reads, 100 * clean_reads / (after / 4)}'
done

kl
cd ${HOME}/neotropical_diversity/data/

VSEARCH="${HOME}/bin/vsearch/bin/vsearch"
TMP_FASTA=$(mktemp)
TMP_FASTA_DEREPLICATED=$(mktemp)
OUTPUT="neotropical_soil_175_samples.fas"

cat ./201[35]*/[LTB][0-9][0-9][0-9]*.fas > ${TMP_FASTA}

# Dereplicate (vsearch)
"${VSEARCH}" --threads 1 \
    --derep_fulllength ${TMP_FASTA} \
    --sizein \
    --sizeout \
    --fasta_width 0 \
    --output ${TMP_FASTA_DEREPLICATED}

# Change abundance annotations
sed -e '/^>/ s/;size=/_/' \
    -e '/^>/ s/;$//' < ${TMP_FASTA_DEREPLICATED} > ${OUTPUT}

bzip2 -9k ${OUTPUT} &

rm ${TMP_FASTA} ${TMP_FASTA_DEREPLICATED}

# kl
cd ${HOME}/src/
# Target the big memory node (1024 GB)
bsub -q normal -n 16 -R "span[hosts=1] select[model==XEON_E5_4650] rusage[mem=200000]" bash swarm_fastidious.sh ../neotropical_diversity/data/neotropical_soil_175_samples.fas

#!/bin/bash -

# Target SSE4.1-able nodes, keep all the threads on one host
# Usage: bsub -q normal-s -n 16 -R "span[hosts=1] select[model==XEON_E5_2670] rusage[mem=32768]" bash swarm.sh target.fas

SWARM="${HOME}/bin/swarm/bin/swarm"
FASTA_FILE=$(readlink -f "${1}")
RESOLUTION="1"
THREADS="16"
OUTPUT_SWARMS="${FASTA_FILE%%.*}_${RESOLUTION}.swarms"
OUTPUT_STATS="${FASTA_FILE%%.*}_${RESOLUTION}.stats"
OUTPUT_STRUCT="${FASTA_FILE%%.*}_${RESOLUTION}.struct"
OUTPUT_REPRESENTATIVES="${FASTA_FILE%%.*}_${RESOLUTION}_representatives.fas"

# check if compressed file
if [[ ${FASTA_FILE##*.} == "bz2" ]] ; then
    bzcat "${FASTA_FILE}" > ${FASTA_FILE/.bz2/}
    FASTA_FILE=${FASTA_FILE/.bz2/}
fi

# Verify the abundance annotation style
ANNOTATION=$(head -n 1 "${FASTA_FILE}" | grep -o ";size=\|_")
case "${ANNOTATION}" in
    ";size=")
        ANNOTATION_OPTION="-z"
        ;;
    "_")
        ANNOTATION_OPTION=""
        ;;
    *)
        echo "Unidentified abundance annotation (\"_\" or \";size=\")." 1>&2
        exit 1
        ;;
esac


# run swarm
if [[ ${ANNOTATION_OPTION} ]] ; then
    "${SWARM}" -d "${RESOLUTION}" \
        -w "${OUTPUT_REPRESENTATIVES}" \
        -i "${OUTPUT_STRUCT}" \
        -t "${THREADS}" "${ANNOTATION_OPTION}" \
        -s "${OUTPUT_STATS}" < "${FASTA_FILE}" > "${OUTPUT_SWARMS}"
else
    "${SWARM}" -d "${RESOLUTION}" \
        -w "${OUTPUT_REPRESENTATIVES}" \
        -i "${OUTPUT_STRUCT}" \
        -t "${THREADS}" \
        -s "${OUTPUT_STATS}" < "${FASTA_FILE}" > "${OUTPUT_SWARMS}"
fi


exit 0

kl
cd ${HOME}/src/
bash stampa.sh ../neotropical_diversity/data/neotropical_soil_175_samples_1f_representatives.fas SSU_V4

kl
cd ${HOME}/neotropical_diversity/data/
awk -F "_" '/^>/ {s += $2 ; c += 1} END {print s, c}' neotropical_soil_175_samples.fas

cd ~/neotropical_diversity/results/stampa/
SOURCE="neotropical_soil_175_samples.OTU.protists_reads.stampa"
awk '{s += $2} END {print s}' "${SOURCE}"

cd ~/neotropical_diversity/results/stampa/
SOURCE="neotropical_soil_175_samples.OTU.fungi_reads.stampa"
awk '{s += $2} END {print s}' "${SOURCE}"

kl

DIR="${HOME}/neotropical_diversity/data/"
SRC="${HOME}/src/"
INPUT="neotropical_soil_175_samples_1f_representatives.fas"
VSEARCH_UCHIME="vsearch_chimera.sh"

# Chimera check with vsearch
cd "${SRC}"
bsub -q long \
    -n 1 \
    -R "select[model==XEON_E5_2670] rusage[mem=20000]" \
    bash "${VSEARCH_UCHIME}" "${DIR}${INPUT}"

#!/bin/bash -

# Target SSE4.1-able nodes, keep all the threads on one host
# Usage: bsub -q long -n 1 -R "select[model==XEON_E5_2670] rusage[mem=16000]" bash vsearch_chimera.sh FASTA

VSEARCH="${HOME}/bin/vsearch/bin/vsearch"
QUERIES=$(readlink -f "${1}")
CHIMERAS=${QUERIES%.*}_chimeras.fas
UCHIME=${QUERIES%.*}.uchime

## Verify the abundance annotations (expect ";size=")
if [[ ! $(head "${QUERIES}" | grep ";size=") ]] ; then
    echo "Abundance annotations must be in usearch's style (;size=)." 1>&2
    echo "Creating a temporary file with modified annotation style." 1>&2
    TMP=${QUERIES%.*}.tmp
    sed -e '/^>/ s/_/;size=/' "${QUERIES}" > "${TMP}"
    QUERIES="${TMP}"
fi

# Chimera detection (vsearch)
"${VSEARCH}" --uchime_denovo "${QUERIES}" \
    --fasta_width 0 \
    --chimeras "${CHIMERAS}" \
    --uchimeout "${UCHIME}"

# Clean temporary file (if any)
[[ ${QUERIES##*.} == "tmp" ]] && rm -f "${QUERIES}"

exit 0

# kl
cd ${HOME}/neotropical_diversity/src/

SCRIPT="taxonomic_profiles.py"
DIR="../data"
OTU_TABLE="neotropical_soil_175_samples.OTU.table"
PROFILES="${OTU_TABLE/.OTU.table/_taxonomic_profiles.csv}"

module load python/latest-2.7 

python "${SCRIPT}" "${DIR}/${OTU_TABLE}" > "${DIR}/${PROFILES}"

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Parse the OTU table, group OTUs assigned to the same taxa per
    sample and produce taxonomic profiles.
"""

from __future__ import print_function

__author__ = "Frédéric Mahé <mahe@rhrk.uni-kl.fr>"
__date__ = "2015/03/25"
__version__ = "$Revision: 1.0"

import os
import sys
import csv

#*****************************************************************************#
#                                                                             #
#                                  Functions                                  #
#                                                                             #
#*****************************************************************************#


def parse_table(input_file):
    """Parse the OTU table."""

    other_domains = set(["Bacteria", "Archaea", "Organelle"])
    unknowns = set(["*", "No_hit", "not_rRNA", "_X"])

    with open(input_file, "rb") as input_file:
        reader = csv.DictReader(input_file, delimiter="\t")
        grand_total = 0
        first_time = True
        non_samples = set(["OTU", "amplicon", "total", "chimera",
                           "identity", "taxonomy", "references"])
        for row in reader:
            # List samples and build the data structure
            if first_time:
                samples = list(set(row.keys()) - non_samples)
                samples = sorted(samples)
                first_time = False
                data = {"Eukaryota": dict(),
                        "Unknown": {sample: 0 for sample in samples},
                        "Organelle": {sample: 0 for sample in samples},
                        "Bacteria": {sample: 0 for sample in samples},
                        "Archaea": {sample: 0 for sample in samples}}

            # Exclude chimeras
            if row["chimera"] == "N" or row["chimera"] == "NA":
                grand_total += int(row["total"])
                if row["taxonomy"] == "No_hit":
                    domain, super_kingdom, kingdom = "No_hit", "No_hit", "No_hit"
                else:
                    domain, super_kingdom, kingdom =  row["taxonomy"].split("|")[0:3]
                if domain == "Eukaryota":
                    if super_kingdom == "*" or super_kingdom == "Eukaryota_X":
                        kingdom = "Unknown Eukaryota"
                    else:
                        if kingdom == "*" or kingdom.endswith("_X"):
                            # Deal with "Taxa_X" situations
                            kingdom = "Unknown " + super_kingdom.split("_X")[0]
                    if kingdom not in data[domain]:
                        data[domain][kingdom] = {sample: 0 for sample in samples}
                    for sample in samples:
                        data[domain][kingdom][sample] += int(row[sample])
                elif domain in other_domains:
                    for sample in samples:
                        data[domain][sample] += int(row[sample])
                elif domain in unknowns:
                    domain = "Unknown"
                    for sample in samples:
                        data[domain][sample] += int(row[sample])
                else:
                    print("Something's wrong! Unknown domain:\n",
                          domain, row, file=sys.stderr)
                    sys.exit(-1)
    return samples, data, grand_total


def output_table(samples, data, grand_total):
    """Output taxonomic profiles"""

    # Taxonomic domains
    non_eucaryotic_domains = ["Archaea", "Bacteria", "Organelle", "Unknown"]

    # Header
    print("taxa", "\t".join(samples),
          "total", "percentage", sep="\t", file=sys.stdout)
    # Other domains
    for domain in non_eucaryotic_domains:
        tmp = [str(data[domain][sample])
               for sample in samples]
        total = sum(data[domain].values())
        percentage = round(100.0 * total / grand_total, 2)
        print(domain, "\t".join(tmp),
              total, percentage, sep="\t", file=sys.stdout)
    # Eukaryota
    kingdoms = sorted(data["Eukaryota"].keys())
    for kingdom in kingdoms:
        tmp = [str(data["Eukaryota"][kingdom][sample])
               for sample in samples]
        total = sum(data["Eukaryota"][kingdom].values())
        percentage = round(100.0 * total / grand_total, 2)
        print(kingdom, "\t".join(tmp),
              total, percentage, sep="\t", file=sys.stdout)
    return


def main():
    """Parse the OTU table and produce taxonomic profiles."""

    # Parse the OTU table
    samples, data, grand_total = parse_table(sys.argv[1])

    # Export taxonomic profiles
    output_table(samples, data, grand_total)

    return


#*****************************************************************************#
#                                                                             #
#                                     Body                                    #
#                                                                             #
#*****************************************************************************#

if __name__ == '__main__':

    main()

sys.exit(0)

kl
cd ${HOME}/neotropical_diversity/data/
awk '{s += $(NF -1)} END {print s}' neotropical_soil_175_samples_taxonomic_profiles.csv
# 130367129
awk '{if ($158 == "N" || $158 == "NA") s += $157} END {print s}' neotropical_soil_175_samples.OTU.table 
# 130367129

library(ggplot2)
library(scales)
library(reshape2)

setwd("~/neotropical_diversity/results/stampa/")

## Neotropical soil samples taxonomic profiles
input <- "neotropical_soil_175_samples_taxonomic_profiles.csv"
threshold <- 0.1

## Import and format data
df <- read.table(input, sep = "\t", header = TRUE)
df <- df %>%
    filter(percentage >= threshold) %>%
    select(-total, -percentage) %>%
    melt(id.vars = c("taxa"))
colnames(df) <- c("taxa", "sample", "abundance")

## Barcharts
ggplot(df, aes(x = sample, y = abundance, fill = taxa)) +
    geom_bar(stat = "identity") +
    scale_x_discrete(limits = rev(levels(df$sample))) +
    scale_y_continuous(labels = comma) +
    xlab("samples") +
    ylab("number of environmental sequences") +
    coord_flip() +
    theme_bw() +
    ggtitle("Tree Line taxonomic profiles (> 0.1%)") +
    theme(legend.justification=c(1,0),
          legend.position=c(1,0),
          legend.background = element_rect(colour="black", size=.1))

## Output to PDF
output <- gsub(".csv", ".pdf", input, fixed = TRUE)
ggsave(file = output, width = 8 , height = 14)

quit(save="no")

library(dplyr)
library(tidyr)
library(ggplot2)
library(scales)
library(reshape2)

## Load data
setwd("~/neotropical_diversity/results/stampa/")
input <- "neotropical_soil_175_samples.OTU.protists.table"

## Multiple plot function
##
## ggplot objects can be passed in ..., or to plotlist (as a list of ggplot objects)
## - cols:   Number of columns in layout
## - layout: A matrix specifying the layout. If present, 'cols' is ignored.
##
## If the layout is something like matrix(c(1,2,3,3), nrow=2, byrow=TRUE),
## then plot 1 will go in the upper left, 2 will go in the upper right, and
## 3 will go all the way across the bottom.
##
multiplot <- function(..., plotlist=NULL, file, cols=1, layout=NULL) {
    library(grid)

    ## Make a list from the ... arguments and plotlist
    plots <- c(list(...), plotlist)

    numPlots = length(plots)

    ## If layout is NULL, then use 'cols' to determine layout
    if (is.null(layout)) {
        ## Make the panel
        ## ncol: Number of columns of plots
        ## nrow: Number of rows needed, calculated from # of cols
        layout <- matrix(seq(1, cols * ceiling(numPlots/cols)),
                         ncol = cols, nrow = ceiling(numPlots/cols))
    }

    if (numPlots==1) {
        print(plots[[1]])

    } else {
          ## Set up the page
          grid.newpage()
          pushViewport(viewport(layout = grid.layout(nrow(layout), ncol(layout))))

          ## Make each plot, in the correct location
          for (i in 1:numPlots) {
              ## Get the i,j matrix positions of the regions that contain this subplot
              matchidx <- as.data.frame(which(layout == i, arr.ind = TRUE))

              print(plots[[i]], vp = viewport(layout.pos.row = matchidx$row,
                                    layout.pos.col = matchidx$col))
          }
      }
}


## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>% tbl_df()

## For some reasons, the command below does not work with mutate
d$Barro <- rowSums(select(d, starts_with("B")))
d$LaSelva <- rowSums(select(d, starts_with("L")))
d$Tiputini <- rowSums(select(d, starts_with("T")) %>%
                      select(-taxonomy, -total))

## Discard all other columns
d <- select(d, one_of("Barro", "Tiputini", "LaSelva", "taxonomy"))

## Extract the third field from the "taxonomy" and store in a new column
d$clade <- apply(d["taxonomy"], 1 , function(x) strsplit(x, "|", fixed = TRUE)[[1]][3])

d$LaSelva <- as.integer(d$LaSelva)
d$Tiputini <- as.integer(d$Tiputini)
d$Barro <- as.integer(d$Barro)

## Group by clade (sum reads)
d2 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
        group_by(clade, forest)
d2$abundance <- as.integer(d2$abundance)
d2 <- tally(d2, wt = abundance, sort = FALSE)

## Replace "*" by "Unknown", and discard "Chimera"
d2 <- d2 %>% filter(clade != "Chimera")
d2$clade[d2$clade == "*"] <- "Unknown"
d2$clade[d2$clade == "Alveolata_X"] <- "Alveolata incertae sedis"
d2$clade[d2$clade == "Amoebozoa_X"] <- "Amoebozoa incertae sedis"
d2$clade[d2$clade == "Stramenopiles_X"] <- "non-Ochrophyta Stramenopiles"

## List clades that have significative abundances
main_taxa <- d2 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(wt = n, sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d2$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## All rows in d2 that have a match in main_taxa
d2 <- semi_join(d2, main_taxa, by = "clade")

## Order the legend
taxa_order_reads<- select(d2, clade) %>% distinct()

#------------------------ Absolute barplots -----------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0)) ##  +
    ## ggtitle("Neotropical Forest Soils: protist communities (175 samples, share > 0.1%)") +
    ## theme(legend.background = element_rect(colour="black", size=.1))

## Barcharts (OTUs)

## Group by clade (sum reads)
d3 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
        group_by(clade, forest) %>%
            filter(abundance != "0") %>%
                tally(sort= TRUE)

## Replace "*" by "Unknown", and discard "Chimera"
d3 <- d3 %>% filter(clade != "Chimera")
d3$clade[d3$clade == "*"] <- "Unknown"
d3$clade[d3$clade == "Alveolata_X"] <- "Alveolata incertae sedis"
d3$clade[d3$clade == "Amoebozoa_X"] <- "Amoebozoa incertae sedis"
d3$clade[d3$clade == "Stramenopiles_X"] <- "non-Ochrophyta Stramenopiles"

## List clades that have significative abundances
main_taxa <- d3 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d3$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## ## All rows in d3 that have a match in main_taxa (in the same time,
## it sorts d3 by decreasing number of OTUs)
d3 <- semi_join(d3, main_taxa, by = "clade")

## Order the legend
taxa_order_OTUs <- select(d3, clade) %>% distinct()

## Barcharts
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_absolute.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()


#-------------------------- Percentage barplots -------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Barcharts (OTUs)
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_relative.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()

quit(save="no")

# aragorn
FOLDER="${HOME}/neotropical_diversity/results"
TABLE="neotropical_soil_175_samples.OTU.protists_cleaned.table"

cd ./stampa/
ln -sf ${FOLDER}/first_155_samples/${TABLE} ${TABLE}

library(dplyr)
library(tidyr)
library(ggplot2)
library(scales)
library(reshape2)

## Load data
setwd("~/neotropical_diversity/results/stampa/")
input <- "neotropical_soil_175_samples.OTU.protists_cleaned.table"

## Multiple plot function
##
## ggplot objects can be passed in ..., or to plotlist (as a list of ggplot objects)
## - cols:   Number of columns in layout
## - layout: A matrix specifying the layout. If present, 'cols' is ignored.
##
## If the layout is something like matrix(c(1,2,3,3), nrow=2, byrow=TRUE),
## then plot 1 will go in the upper left, 2 will go in the upper right, and
## 3 will go all the way across the bottom.
##
multiplot <- function(..., plotlist=NULL, file, cols=1, layout=NULL) {
    library(grid)

    ## Make a list from the ... arguments and plotlist
    plots <- c(list(...), plotlist)

    numPlots = length(plots)

    ## If layout is NULL, then use 'cols' to determine layout
    if (is.null(layout)) {
        ## Make the panel
        ## ncol: Number of columns of plots
        ## nrow: Number of rows needed, calculated from # of cols
        layout <- matrix(seq(1, cols * ceiling(numPlots/cols)),
                         ncol = cols, nrow = ceiling(numPlots/cols))
    }

    if (numPlots==1) {
        print(plots[[1]])

    } else {
          ## Set up the page
          grid.newpage()
          pushViewport(viewport(layout = grid.layout(nrow(layout), ncol(layout))))

          ## Make each plot, in the correct location
          for (i in 1:numPlots) {
              ## Get the i,j matrix positions of the regions that contain this subplot
              matchidx <- as.data.frame(which(layout == i, arr.ind = TRUE))

              print(plots[[i]], vp = viewport(layout.pos.row = matchidx$row,
                                    layout.pos.col = matchidx$col))
          }
      }
}


## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>% tbl_df()

## For some reasons, the command below does not work with mutate
d$Barro <- rowSums(select(d, starts_with("B")))
d$LaSelva <- rowSums(select(d, starts_with("L")))
d$Tiputini <- rowSums(select(d, starts_with("T")) %>%
                      select(-taxonomy, -total))

## Discard all other columns
d <- select(d, one_of("Barro", "Tiputini", "LaSelva", "taxonomy"))

## Extract the third field from the "taxonomy" and store in a new column
d$clade <- apply(d["taxonomy"], 1 , function(x) strsplit(x, "|", fixed = TRUE)[[1]][3])

d$LaSelva <- as.integer(d$LaSelva)
d$Tiputini <- as.integer(d$Tiputini)
d$Barro <- as.integer(d$Barro)

## Group by clade (sum reads)
d2 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
        group_by(clade, forest)
d2$abundance <- as.integer(d2$abundance)
d2 <- tally(d2, wt = abundance, sort = FALSE)

## Replace "*" by "Unknown", and discard "Chimera"
d2 <- d2 %>% filter(clade != "Chimera")
d2$clade[d2$clade == "*"] <- "Unknown"
d2$clade[d2$clade == "Alveolata_X"] <- "Alveolata incertae sedis"
d2$clade[d2$clade == "Amoebozoa_X"] <- "Amoebozoa incertae sedis"
d2$clade[d2$clade == "Stramenopiles_X"] <- "non-Ochrophyta Stramenopiles"

## List clades that have significative abundances
main_taxa <- d2 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(wt = n, sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d2$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## All rows in d2 that have a match in main_taxa
d2 <- semi_join(d2, main_taxa, by = "clade")

## Order the legend
taxa_order_reads<- select(d2, clade) %>% distinct()

#------------------------ Absolute barplots -----------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0)) ##  +

## Barcharts (OTUs)

## Group by clade (sum reads)
d3 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
    group_by(clade, forest) %>%
    filter(abundance != "0") %>%
    tally(sort= TRUE)

## Replace "*" by "Unknown", and discard "Chimera"
d3 <- d3 %>% filter(clade != "Chimera")
d3$clade[d3$clade == "*"] <- "Unknown"
d3$clade[d3$clade == "Alveolata_X"] <- "Alveolata incertae sedis"
d3$clade[d3$clade == "Amoebozoa_X"] <- "Amoebozoa incertae sedis"
d3$clade[d3$clade == "Stramenopiles_X"] <- "non-Ochrophyta Stramenopiles"

## List clades that have significative abundances
main_taxa <- d3 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d3$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## ## All rows in d3 that have a match in main_taxa (in the same time,
## it sorts d3 by decreasing number of OTUs)
d3 <- semi_join(d3, main_taxa, by = "clade")

## Order the legend
taxa_order_OTUs <- select(d3, clade) %>% distinct()

## Barcharts
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_absolute.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()


#-------------------------- Percentage barplots -------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Barcharts (OTUs)
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_relative.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()

quit(save="no")

library(dplyr)
library(tidyr)
library(ggplot2)
library(scales)
library(reshape2)

## Load data
setwd("~/neotropical_diversity/results/stampa/")
input <- "neotropical_soil_175_samples.OTU.fungi.table"

## Multiple plot function
##
## ggplot objects can be passed in ..., or to plotlist (as a list of ggplot objects)
## - cols:   Number of columns in layout
## - layout: A matrix specifying the layout. If present, 'cols' is ignored.
##
## If the layout is something like matrix(c(1,2,3,3), nrow=2, byrow=TRUE),
## then plot 1 will go in the upper left, 2 will go in the upper right, and
## 3 will go all the way across the bottom.
##
multiplot <- function(..., plotlist=NULL, file, cols=1, layout=NULL) {
    library(grid)

    ## Make a list from the ... arguments and plotlist
    plots <- c(list(...), plotlist)

    numPlots = length(plots)

    ## If layout is NULL, then use 'cols' to determine layout
    if (is.null(layout)) {
        ## Make the panel
        ## ncol: Number of columns of plots
        ## nrow: Number of rows needed, calculated from # of cols
        layout <- matrix(seq(1, cols * ceiling(numPlots/cols)),
                         ncol = cols, nrow = ceiling(numPlots/cols))
    }

    if (numPlots==1) {
        print(plots[[1]])

    } else {
          ## Set up the page
          grid.newpage()
          pushViewport(viewport(layout = grid.layout(nrow(layout), ncol(layout))))

          ## Make each plot, in the correct location
          for (i in 1:numPlots) {
              ## Get the i,j matrix positions of the regions that contain this subplot
              matchidx <- as.data.frame(which(layout == i, arr.ind = TRUE))

              print(plots[[i]], vp = viewport(layout.pos.row = matchidx$row,
                                    layout.pos.col = matchidx$col))
          }
      }
}


## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>% tbl_df()

## For some reasons, the command below does not work with mutate
d$Barro <- rowSums(select(d, starts_with("B")))
d$LaSelva <- rowSums(select(d, starts_with("L")))
d$Tiputini <- rowSums(select(d, starts_with("T")) %>%
                      select(-taxonomy, -total))

## Discard all other columns
d <- select(d, one_of("Barro", "Tiputini", "LaSelva", "taxonomy"))

## Extract the fourth field from the "taxonomy" and store in a new column
d$clade <- apply(d["taxonomy"], 1 , function(x) strsplit(x, "|", fixed = TRUE)[[1]][4])

d$LaSelva <- as.integer(d$LaSelva)
d$Tiputini <- as.integer(d$Tiputini)
d$Barro <- as.integer(d$Barro)

## Group by clade (sum reads)
d2 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
        group_by(clade, forest)
d2$abundance <- as.integer(d2$abundance)
d2 <- tally(d2, wt = abundance, sort = FALSE)

## Replace "*" by "Unknown", and discard "Chimera"
d2 <- d2 %>% filter(clade != "Chimera")
d2$clade[d2$clade == "*"] <- "Unknown"
d2$clade[d2$clade == "Alveolata_X"] <- "Alveolata"
d2$clade[d2$clade == "Amoebozoa_X"] <- "Amoebozoa"
d2$clade[d2$clade == "Stramenopiles_X"] <- "Stramenopiles"

## List clades that have significative abundances
main_taxa <- d2 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(wt = n, sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d2$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## All rows in d2 that have a match in main_taxa
d2 <- semi_join(d2, main_taxa, by = "clade")

## Order the legend
taxa_order_reads<- select(d2, clade) %>% distinct()

#------------------------ Absolute barplots -----------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0)) ##  +
    ## ggtitle("Neotropical Forest Soils: fungi communities (175 samples, share > 0.1%)") +
    ## theme(legend.background = element_rect(colour="black", size=.1))

## Barcharts (OTUs)

## Group by clade (sum reads)
d3 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
        group_by(clade, forest) %>%
            filter(abundance != "0") %>%
                tally(sort= TRUE)

## Replace "*" by "Unknown", and discard "Chimera"
d3 <- d3 %>% filter(clade != "Chimera")
d3$clade[d3$clade == "*"] <- "Unknown"
d3$clade[d3$clade == "Alveolata_X"] <- "Alveolata"
d3$clade[d3$clade == "Amoebozoa_X"] <- "Amoebozoa"
d3$clade[d3$clade == "Stramenopiles_X"] <- "Stramenopiles"

## List clades that have significative abundances
main_taxa <- d3 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d3$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## ## All rows in d3 that have a match in main_taxa (in the same time,
## it sorts d3 by decreasing number of OTUs)
d3 <- semi_join(d3, main_taxa, by = "clade")

## Order the legend
taxa_order_OTUs <- select(d3, clade) %>% distinct()

## Barcharts
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_absolute.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()


#-------------------------- Percentage barplots -------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_reads$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Barcharts (OTUs)
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order_OTUs$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table", "_group_by_forests_relative.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()

quit(save="no")

# aragorn
cd ~/neotropical_diversity/results/stampa/
TABLE="neotropical_soil_175_samples.OTU.fungi.table"
sed 's/Ascomycota_X|Ascomycota_XX|Archaeorhizomyces/Archaeorhizomyces|Archaeorhizomyces|Archaeorhizomyces/' ${TABLE}| \
sed 's/Fungi_XX/Unknown/' > "${TABLE}.tmp"

library(dplyr)
library(tidyr)
library(ggplot2)
library(scales)
library(reshape2)

## Load data
setwd("~/neotropical_diversity/results/stampa/")
input <- "neotropical_soil_175_samples.OTU.fungi.table.tmp"

## Multiple plot function
##
## ggplot objects can be passed in ..., or to plotlist (as a list of ggplot objects)
## - cols:   Number of columns in layout
## - layout: A matrix specifying the layout. If present, 'cols' is ignored.
##
## If the layout is something like matrix(c(1,2,3,3), nrow=2, byrow=TRUE),
## then plot 1 will go in the upper left, 2 will go in the upper right, and
## 3 will go all the way across the bottom.
##
multiplot <- function(..., plotlist=NULL, file, cols=1, layout=NULL) {
    library(grid)

    ## Make a list from the ... arguments and plotlist
    plots <- c(list(...), plotlist)

    numPlots = length(plots)

    ## If layout is NULL, then use 'cols' to determine layout
    if (is.null(layout)) {
        ## Make the panel
        ## ncol: Number of columns of plots
        ## nrow: Number of rows needed, calculated from # of cols
        layout <- matrix(seq(1, cols * ceiling(numPlots/cols)),
                         ncol = cols, nrow = ceiling(numPlots/cols))
    }

    if (numPlots==1) {
        print(plots[[1]])

    } else {
          ## Set up the page
          grid.newpage()
          pushViewport(viewport(layout = grid.layout(nrow(layout), ncol(layout))))

          ## Make each plot, in the correct location
          for (i in 1:numPlots) {
              ## Get the i,j matrix positions of the regions that contain this subplot
              matchidx <- as.data.frame(which(layout == i, arr.ind = TRUE))

              print(plots[[i]], vp = viewport(layout.pos.row = matchidx$row,
                                    layout.pos.col = matchidx$col))
          }
      }
}


## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>% tbl_df()

## For some reasons, the command below does not work with mutate
d$Barro <- rowSums(select(d, starts_with("B")))
d$LaSelva <- rowSums(select(d, starts_with("L")))
d$Tiputini <- rowSums(select(d, starts_with("T")) %>%
                      select(-taxonomy, -total))

## Discard all other columns
d <- select(d, one_of("Barro", "Tiputini", "LaSelva", "taxonomy"))

## Extract the fifth field from the "taxonomy" and store in a new column
d$clade <- apply(d["taxonomy"], 1 , function(x) strsplit(x, "|", fixed = TRUE)[[1]][5])
d$LaSelva <- as.integer(d$LaSelva)
d$Tiputini <- as.integer(d$Tiputini)
d$Barro <- as.integer(d$Barro)

## Group by clade (sum reads)
d2 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
    group_by(clade, forest)
d2$abundance <- as.integer(d2$abundance)
d2 <- tally(d2, wt = abundance, sort = FALSE)

## Replace "*" by "Unknown", and discard "Chimera"
d2 <- d2 %>% filter(clade != "Chimera")
d2$clade[d2$clade == "*"] <- "Unknown"

## List clades that have significative abundances
main_taxa <- d2 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(wt = n, sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d2$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## All rows in d2 that have a match in main_taxa
d2 <- semi_join(d2, main_taxa, by = "clade") %>%
    arrange(clade)

## Barcharts (OTUs)
## Group by clade (sum reads)
d3 <- select(d, -taxonomy) %>%
    gather("forest", "abundance", -clade) %>%
    group_by(clade, forest) %>%
    filter(abundance != "0") %>%
    tally(sort= TRUE)

## Replace "*" by "Unknown", and discard "Chimera"
d3 <- d3 %>% filter(clade != "Chimera")
d3$clade[d3$clade == "*"] <- "Unknown"
## d3$clade[d3$clade == "Taphrinomycotina"] <- "Taphrinomycotina (excluding the Archaeorhizomycetes)"

## List clades that have significative abundances
main_taxa <- d3 %>%
    select(-forest) %>%
    group_by(clade) %>%
    tally(sort = TRUE) %>%
    mutate(percentage = 100 * n / sum(d3$n)) %>%
    filter(percentage > 0.1) %>%
    select(-n, -percentage)

## ## All rows in d3 that have a match in main_taxa
d3 <- semi_join(d3, main_taxa, by = "clade") %>% arrange(clade)

## Order the legend
taxa_order <- bind_rows(select(d2, clade), select(d3, clade)) %>% arrange(clade) %>% distinct()

#------------------------ Absolute barplots -----------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Barcharts (OTUs)
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity") +
    scale_y_continuous(labels = comma) +
    scale_fill_discrete(breaks = taxa_order$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("number of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table.tmp", "_5th_level_group_by_forests_absolute.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()


#-------------------------- Percentage barplots -------------------------------#

## Barcharts (reads)
p1 <- ggplot(d2, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order$clade,
                            name = "clade                         ") +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed reads") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Barcharts (OTUs)
p2 <- ggplot(d3, aes(x = forest, y = n, fill = clade)) +
    geom_bar(stat = "identity", position = "fill") +
    scale_y_continuous(labels = percent_format()) +
    scale_fill_discrete(breaks = taxa_order$clade) +
    scale_x_discrete(limits = rev(c("LaSelva", "Barro", "Tiputini")),
                     breaks = c("LaSelva", "Barro", "Tiputini"),
                     labels = c("Costa Rica\n(La Selva)",
                         "Panama\n(Barro\nColorado)",
                         "Ecuador\n(Tiputini)")) +
    ylab("percentage of observed OTUs") +
    xlab("forests") +
    coord_flip() +
    theme_bw() +
    theme(axis.text  = element_text(size = 11),
          axis.title.x = element_text(vjust = 0))

## Output to PDF (multiplot)
output <- gsub(".table.tmp", "_5th_level_group_by_forests_relative.pdf", input, fixed = TRUE)
pdf(file = output, width = 11 , height = 10)
multiplot(p1, p2)
dev.off()

quit(save="no")

# aragorn
cd ~/neotropical_diversity/results/stampa/
# clean
TABLE="neotropical_soil_175_samples.OTU.fungi.table"
rm -f "${TABLE}.tmp"

library(dplyr)
library(tidyr)
library(vegan)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists_cleaned.table"

## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>%
    tbl_df() %>%
    select(total)

## transpose (to get OTUs in columns)
d <- t(d)

## -------------------------------------------------------------------------- ##
## Alpha diversity (global and per sample)

## richness
estimateR(d)

## Shannon index H (richness + evenness)
H <- diversity(d, index = "shannon", MARGIN = 1, base = exp(1))
H

## Pielou’s index of evenness: (0-1, 1 = max. evenness)
J <- H/log(dim(d)[2])
J

## Simpson's D index: (richness + evenness, 0-1; 1 - D rises as evenness increases)
D <- diversity(d, "simpson")
D
inv_D <- diversity(d, "invsimpson")
inv_D

quit(save = "no")

library(dplyr)
library(tidyr)
library(breakaway)

## Load data
setwd("~/neotropical_diversity/results/stampa/")
input <- "neotropical_soil_175_samples.OTU.protists_cleaned.table"

## Import and format data
d <- read.table(input, sep = "\t", header = TRUE, dec=".") %>% tbl_df()

## For some reasons, the command below does not work with mutate
d$Barro <- rowSums(select(d, starts_with("B")))
d$LaSelva <- rowSums(select(d, starts_with("L")))
d$Tiputini <- rowSums(select(d, starts_with("T")) %>%
                      select(-taxonomy, -total))

## Discard all other columns
d <- select(d, one_of("Barro", "Tiputini", "LaSelva", "total"))

## Write to file
write.csv(file = "neotropical_soil_175_samples.OTU.protists_cleaned_forests.table", x = d)

quit(save="no")

cd ~/neotropical_diversity/results/stampa/

BREAKAWAY="../../src/breakaway.R"
TABLE="neotropical_soil_175_samples.OTU.protists_cleaned_forests.table"
OUTPUT="${TABLE/.table/.alphadiversity.table}"
TMP=$(mktemp)

# Loop over the samples
echo -e "SAMPLE\tMASS\tOBSERVED\tESTIMATES\tSE" > "${OUTPUT}"
for ((i=2 ; i<=5 ; i++)) ; do
    COLUMN=$(cut -d "," -f ${i} "${TABLE}" | grep -v "^0$")
    SAMPLE=$(head -n 1 <<< "${COLUMN}" | tr -d "\"")
    MASS=$(awk '{s += $1} END {print s}' <<< "${COLUMN}")
    OBSERVED=$(wc -l <<< "${COLUMN}")
    OBSERVED=$(( $OBSERVED - 1 ))
    tail -n +2 <<< "${COLUMN}" | sort -n | uniq -c | \
        awk '{print $2, $1}' > "${TMP}"
    ESTIMATES=""
    ESTIMATES=$(Rscript "${BREAKAWAY}" "${TMP}" | cut -d " " -f 2 | paste - -)
    [[ ! "${ESTIMATES}" ]] && ESTIMATES=$(echo -e "NA\tNA")
    echo -e "${SAMPLE}\t${MASS}\t${OBSERVED}\t${ESTIMATES}"
done >> "${OUTPUT}"

rm "${TMP}"

#!/usr/bin/Rscript

library(breakaway)

setwd("~/neotropical_diversity/results/stampa/")
args <- commandArgs(trailingOnly = TRUE)
input <- args[1]

d <- read.table(input, sep = " ")

answers <- breakaway(d, plot = FALSE, print = FALSE,
                     answers = TRUE, force = FALSE)

round(answers$est)
round(answers$seest)

quit(save="no")

#' ---
#' title: "Neotropical Forests breakaway and betta analysis"
#' author: "Sarah Sernaker"
#' ---
# install.packages('breakaway')

library('breakaway')

datTab <- read.table("neotropical_soil_175_samples.OTU.protists_cleaned_forests_recleaned.table",
                     header = TRUE, sep=",")
names<-colnames(datTab)


names<-names[-1:-1]
datTab<-datTab[,-1:-1]  # take off first column


freqTab <- lapply(apply(datTab,2,table),as.data.frame) 
freqTab <- lapply(freqTab,function(x) x[x[,1]!=0,])  # create frequency vector f_i= #(i)

totalSpecies <- unlist(lapply(freqTab,function(x) sum(x[,2]))) # sum of observed species

estimates_baway_newdata <- matrix(NA,nrow=length(freqTab),ncol=4)
rownames(estimates_baway_newdata) <- names(freqTab)
colnames(estimates_baway_newdata) <- c("baway_est","baway_seest","baway_lcb","baway_ucb")

for (i in 1:length(freqTab)) {  # breakaway ests of total species, std error, and 95% C.I
    baway1 <- try(breakaway(freqTab[[i]],plot=FALSE,print=FALSE,answers=TRUE), silent=T)
    if(!is.null(baway1$est)) { # if it works, store it
      estimates_baway_newdata[i,1] <- baway1$est
      estimates_baway_newdata[i,2] <- baway1$seest
      estimates_baway_newdata[i,3:4] <- baway1$ci
    }
}
estimates_baway_newdata

# write.table(estimates_baway_newdata, "bwayEsts_total.csv", sep=",")

# apply betta function using breakaway total estimates as initial estimates
bettaEsts<-betta(estimates_baway_newdata[,1],estimates_baway_newdata[,2])

# plot with breakaway output
# pdf("confInts_breakaway.pdf",height=6,width=10)
plot(0,0,type="n",bty="n",main="95% confidence intervals of estimated diversity"
     ,xlab=" ", ylab="breakaway estimates of total diversity", xaxt='n', xlim=c(.5,4), 
     ylim=c(0,30000))
for (i in 1:4){
  lines(c(i,i),c(estimates_baway_newdata[i,3],estimates_baway_newdata[i,4]))
  points(i,estimates_baway_newdata[i,1], pch='-')
}
axis(1,1:length(bettaEsts$blups),tick=F,labels=rownames(estimates_baway_newdata),
     las=3,cex.axis=1)
# dev.off()

# plot with betta output
# pdf("confInts_betta.pdf",height=6,width=10)
conf_ints <-matrix(rep(0), length(bettaEsts$blups),2)
rownames(conf_ints) <- rownames(estimates_baway_newdata)
colnames(conf_ints)<-c("Lower", "Upper")
plot(0,0,type="n",bty="n",main="95% confidence intervals of estimated diversity"
     ,xlab=" ",ylab="betta estimates of total diversity", xaxt='n', xlim=c(.5,4), 
     ylim=c(0,30000))
for (i in 1:length(bettaEsts$blups)){ #length(bettaEsts$blups)
  conf_ints[i,1]<-max(0,bettaEsts$blups[i]-1.96*bettaEsts$blupses[i])
  conf_ints[i,2]<-bettaEsts$blups[i]+1.96*bettaEsts$blupses[i]
  lines(c(i,i),c(conf_ints[i,1],conf_ints[i,2]))
  points(i,bettaEsts$blups[i], pch='-')
}
axis(1,1:length(bettaEsts$blups),tick=F,labels=rownames(conf_ints),las=3,cex.axis=1)
dev.off()
# write.table(conf_ints, "betta_confints.csv", sep=",")

library(vegan)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE, row.names = 1)

## -------------------------------------------------------------------------- ##
## Cleaning

## Samples with less than 10,000 reads
## B199_B200, B175_B176, L111_L112, B197_B198, B145_B146, B167_B168, L183_L184, L131_L132, L097_L098, L181_L182
## Outliers
## B173_B174  # Low diversity, mostly made of one OTU of Chlorophyceae Dunaliella

## Reduce (remove useless columns and low samples)
d <- subset(d,
            select = -c(amplicon, total, chimera, identity,
                        taxonomy, references,               
                        B199_B200, B175_B176, L111_L112, B197_B198,
                        B145_B146, B167_B168, L183_L184, L131_L132,
                        L097_L098, L181_L182,
                        B173_B174))

## Identify low samples and outliers
quantile(rowSums(d))
rowSums(d)
min(rowSums(d))
hist(rowSums(d))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))


## -------------------------------------------------------------------------- ##
## Alpha diversity (global and per sample)

## richness
estimateR(d_rarefied)

## Alpha diversity: evenness
plot(colSums(d), log = "y", xlab = "Rank", ylab = "abundance")  # global
mod <- radfit(d)  # (per sample diversity)
plot(mod)
output_file <- paste(output, "_evenness.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## ## Accumulation curves (using Ugland’s method) (global)
## CC_CURVugland <- specaccum(d, method = "exact", permutations=1000)
## plot(CC_CURVugland)
## output_file <- paste(output, "_ugland.pdf", sep = "")
## dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## ## Preston model (global)
## preston <- prestonfit(colSums(d))
## prestondistr <- prestondistr(colSums(d))
## plot(preston)
## lines(prestondistr, line.col = "blue3")
## output_file <- paste(output, "_preston.pdf", sep = "")
## dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## Shannon index H (richness + evenness)
H <- diversity(d, index = "shannon", MARGIN = 1, base = exp(1))
H

## Pielou’s index of evenness: (0-1, 1 = max. evenness)
J <- H/log(dim(d)[2])
J

## Simpson's D index: (richness + evenness, 0-1; 1 - D rises as evenness increases)
D <- diversity(d, "simpson")
D
inv_D <- diversity(d, "invsimpson")
inv_D

## Rényi diversities
R <- renyi(d_rarefied)
plot(R)
output_file <- paste(output, "_renyi.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## Rarefaction curves
## rarecurve(d, step = 100000, xlab = "sample size (number of reads)", ylab = "number of OTUs")
## output_file <- paste(output, "_rarefaction_curves.pdf", sep = "")
## dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)


## -------------------------------------------------------------------------- ##
## Beta diversity

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")
d_rarefied.bray

## NMDS analyses
d_rarefied.bray.nmds <- monoMDS(d_rarefied.bray)
d_rarefied.bray.nmds
plot(d_rarefied.bray.nmds)
output_file <- paste(output, "_NMDS.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)
stressplot(d_rarefied.bray.nmds)
output_file <- paste(output, "_NMDS_stress.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## UPGMA (Unweighted Pair Group Method with Arithmetic Mean)
d_rarefied.bray.hclust <- hclust(d_rarefied.bray, "average")
plot(d_rarefied.bray.hclust)
output_file <- paste(output, "_UPGMA.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)
d_rarefied.bray.hclust

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(scales)
library(grid)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE, row.names = 1)

## -------------------------------------------------------------------------- ##
## Cleaning

## Samples with less than 10,000 reads
## B199_B200, B175_B176, L111_L112, B197_B198, B145_B146, B167_B168, L183_L184, L131_L132, L097_L098, L181_L182
## Outliers
## B173_B174  # Low diversity, mostly made of one OTU of Chlorophyceae Dunaliella

## Reduce (remove useless columns and low samples)
d <- subset(d,
            select = -c(amplicon, total, chimera, identity,
                        taxonomy, references,               
                        B199_B200, B175_B176, L111_L112, B197_B198,
                        B145_B146, B167_B168, L183_L184, L131_L132,
                        L097_L098, L181_L182,
                        B173_B174))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## -------------------------------------------------------------------------- ##
## Beta diversity

## Jaccard dissimilarity matrix
d_rarefied.jaccard <- vegdist(d_rarefied, method = "jaccard")
jaccard <- as.matrix(as.dist(d_rarefied.jaccard))
output_file <- paste(output, "_jaccard.csv", sep = "")
write.table(jaccard, file = output_file, sep = "\t", row.names = TRUE)

## colnames(d) <- c("sampleA", "sampleB", "sizeA", "sizeB", "commonA", "commonB")
jaccard2 <- as.data.frame(jaccard)
jaccard2 <- tbl_df(jaccard2)
samples <- as.data.frame(colnames(jaccard2))
colnames(samples) <- c("sampleA")
jaccard3 <- bind_cols(samples, jaccard2)
jaccard3 <- gather(jaccard3, "sampleB", "distance", 2:ncol(jaccard3)) %>%
    filter(distance > 0) %>%
    mutate(similarity = 1 - distance)

breaks <- which(levels(jaccard3$sampleA) %in% c("B193_B194", "L199_L200"))

## Plot
ggplot(jaccard3, aes(x = sampleA, y = sampleB)) +
    theme_bw() +
    geom_tile(aes(fill = similarity), colour = "white") +
    scale_fill_gradient(low = "white", high = "steelblue") +
    theme(axis.text.x = element_text(size = 5, angle = 270, vjust = 0.5, hjust = 0),
          axis.text.y = element_text(size = 5),
          panel.grid.minor = element_blank(),
          panel.grid.major = element_blank(),
          panel.background = element_blank(),
          axis.title.x = element_blank(),
          axis.title.y = element_blank()) +
     geom_vline(size = 0.1, xintercept = breaks + c(0.5, 0.5)) +
     geom_hline(size = 0.1, yintercept = breaks + c(0.5, 0.5))

## Output file
output_file <- paste(output, "_jaccard.pdf", sep = "")
ggsave(file = output_file, width = 12 , height = 10)

quit(save = "no")

library(vegan)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.fungi.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE)

## -------------------------------------------------------------------------- ##
## Cleaning

## Samples with less than 2,000 reads
## Outliers
## B005_B006  # Low diversity?

d <- subset(d,
            select = -c(OTU, amplicon, total, chimera, identity,
                        taxonomy, references,
                        B155_B156, B173_B174, B013_B014, B133_B134,
                        L111_L112, B167_B168, B197_B198, B007_B008,
                        T195_T196, T174, B031_B032,
                        B005_B006))

## transpose (to get OTUs in columns)
d <- t(d)

## Identify low samples and outliers
quantile(rowSums(d))
sort(rowSums(d))[1:20]
min(rowSums(d))
hist(rowSums(d))

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## -------------------------------------------------------------------------- ##
## Alpha diversity (global and per sample)

## richness
estimateR(d_rarefied)

## Alpha diversity: evenness
plot(colSums(d), log = "y", xlab = "Rank", ylab = "abundance")  # global
mod <- radfit(d)  # (per sample diversity)
plot(mod)
output_file <- paste(output, "_evenness.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## ## Accumulation curves (using Ugland’s method) (global)
## CC_CURVugland <- specaccum(d, method = "exact", permutations=1000)
## plot(CC_CURVugland)
## output_file <- paste(output, "_ugland.pdf", sep = "")
## dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## ## Preston model (global)
## preston <- prestonfit(colSums(d))
## prestondistr <- prestondistr(colSums(d))
## plot(preston)
## lines(prestondistr, line.col = "blue3")
## output_file <- paste(output, "_preston.pdf", sep = "")
## dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## Shannon index H (richness + evenness)
H <- diversity(d, index = "shannon", MARGIN = 1, base = exp(1))
H

## Pielou’s index of evenness: (0-1, 1 = max. evenness)
J <- H/log(dim(d)[2])
J

## Simpson's D index: (richness + evenness, 0-1; 1 - D rises as evenness increases)
D <- diversity(d, "simpson")
D
inv_D <- diversity(d, "invsimpson")
inv_D

## Rényi diversities
R <- renyi(d_rarefied)
plot(R)
output_file <- paste(output, "_renyi.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## Rarefaction curves
rarecurve(d, step = 100000, xlab = "sample size (number of reads)", ylab = "number of OTUs")
output_file <- paste(output, "_rarefaction_curves.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)


## -------------------------------------------------------------------------- ##
## Beta diversity

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")
d_rarefied.bray

## NMDS analyses
d_rarefied.bray.nmds <- monoMDS(d_rarefied.bray)
d_rarefied.bray.nmds
plot(d_rarefied.bray.nmds)
output_file <- paste(output, "_NMDS.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)
stressplot(d_rarefied.bray.nmds)
output_file <- paste(output, "_NMDS_stress.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)

## UPGMA (Unweighted Pair Group Method with Arithmetic Mean)
d_rarefied.bray.hclust <- hclust(d_rarefied.bray, "average")
plot(d_rarefied.bray.hclust)
output_file <- paste(output, "_UPGMA.pdf", sep = "")
dev.copy2pdf(device = x11, file = output_file, out.type = "pdf", width = 12, height = 6)
d_rarefied.bray.hclust

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(scales)
library(grid)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.fungi.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE)

## -------------------------------------------------------------------------- ##
## Cleaning

## Samples with less than 2,000 reads
## Outliers
## B005_B006  # Low diversity

## Reduce (remove useless columns and low samples)
d <- subset(d,
            select = -c(OTU, amplicon, total, chimera, identity,
                        taxonomy, references,
                        B155_B156, B173_B174, B013_B014, B133_B134,
                        L111_L112, B167_B168, B197_B198, B007_B008,
                        T195_T196, T174, B031_B032,
                        B005_B006))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## -------------------------------------------------------------------------- ##
## Beta diversity

## Jaccard dissimilarity matrix
d_rarefied.jaccard <- vegdist(d_rarefied, method = "jaccard")
jaccard <- as.matrix(as.dist(d_rarefied.jaccard))
output_file <- paste(output, "_jaccard.csv", sep = "")
write.table(jaccard, file = output_file, sep = "\t", row.names = TRUE)

## colnames(d) <- c("sampleA", "sampleB", "sizeA", "sizeB", "commonA", "commonB")
jaccard2 <- as.data.frame(jaccard)
jaccard2 <- tbl_df(jaccard2)
samples <- as.data.frame(colnames(jaccard2))
colnames(samples) <- c("sampleA")
jaccard3 <- bind_cols(samples, jaccard2)
jaccard3 <- gather(jaccard3, "sampleB", "distance", 2:ncol(jaccard3)) %>%
    filter(distance > 0) %>%
    mutate(similarity = 1 - distance)

breaks <- which(levels(jaccard3$sampleA) %in% c("B193_B194", "L199_L200"))

## Plot
ggplot(jaccard3, aes(x = sampleA, y = sampleB)) +
    theme_bw() +
    geom_tile(aes(fill = similarity), colour = "white") +
    scale_fill_gradient(low = "white", high = "steelblue") +
    expand_limits(fill = c(0, 1)) +
    theme(axis.text.x = element_text(size = 5, angle = 270, vjust = 0.5, hjust = 0),
          axis.text.y = element_text(size = 5),
          legend.title = element_text(face = "bold"),
          panel.grid.minor = element_blank(),
          panel.grid.major = element_blank(),
          panel.background = element_blank(),
          axis.title.x = element_blank(),
          axis.title.y = element_blank()) +
     geom_vline(size = 0.1, xintercept = breaks + c(0.5, 0.5)) +
     geom_hline(size = 0.1, yintercept = breaks + c(0.5, 0.5))

## Output file
output_file <- paste(output, "_jaccard.pdf", sep = "")
ggsave(file = output_file, width = 12 , height = 10)

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(ggrepel)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists.table"
output <- gsub(".table", "", input, fixed = TRUE)
iterations <- 3e4

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE, row.names = 1)

## -------------------------------------------------------------------------- ##
## Cleaning
## Samples with less than 10,000 reads
## B199_B200, B175_B176, L111_L112, B197_B198, B145_B146, B167_B168, L183_L184, L131_L132, L097_L098, L181_L182
## Outliers
## B173_B174  # Low diversity, mostly made of one OTU of Chlorophyceae Dunaliella

## Reduce (remove useless columns and low samples)
d <- subset(d,
            select = -c(amplicon, total, chimera, identity,
                        taxonomy, references,               
                        B199_B200, B175_B176, L111_L112, B197_B198,
                        B145_B146, B167_B168, L183_L184, L131_L132,
                        L097_L098, L181_L182,
                        B173_B174))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")

## NMDS analyses
d_rarefied.bray.nmds <- monoMDS(d_rarefied.bray)

## Extract data scores (https://chrischizinski.github.io/rstats/2014/04/13/vegan-ggplot2/)
stress <- d_rarefied.bray.nmds$stress
samples <- rownames(d_rarefied)
data.scores <- as.data.frame(scores(d_rarefied.bray.nmds))
data.scores$site <- rownames(data.scores)
data.scores$samples <- samples
head(data.scores)

## Add a forest variable
data.scores <- mutate(data.scores, forest = substr(site, 1, 1)) %>% select(-site)
data.scores$forest[data.scores$forest == "B"] <- "Panama (Barro Colorado)"
data.scores$forest[data.scores$forest == "L"] <- "Costa Rica (La Selva)"
data.scores$forest[data.scores$forest == "T"] <- "Ecuador (Tiputini)"
x_min <- min(data.scores$MDS1)
y_max <- max(data.scores$MDS2)
stress_annotation <- paste("stress: ", round(stress, digits = 4), sep = "")
head(data.scores)

## Plot
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, label = samples)) +
    geom_text_repel(alpha = 0.5,
                    size = 2.75,
                    segment.size = 0.25,
                    segment.color = "grey",
                    max.iter = iterations) +
    geom_point(aes(colour = forest), size = 2) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    annotate("text", x = x_min + abs(x_min / 10), y = y_max, label = stress_annotation) +
    coord_equal()

output_file <- paste(output, "_MDS.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

## Plot (no sample names)
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, colour = forest)) +
    geom_point(size = 3) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    annotate("text", x = x_min + abs(x_min / 10), y = y_max, label = stress_annotation) +
    coord_equal()

output_file <- paste(output, "_MDS_no_labels.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(ggrepel)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.fungi.table"
output <- gsub(".table", "", input, fixed = TRUE)
iterations <- 3e4

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE)

## -------------------------------------------------------------------------- ##
## Cleaning
## Samples with less than 2,000 reads
## Outliers
## B005_B006  # Low diversity?

d <- subset(d,
            select = -c(OTU, amplicon, total, chimera, identity,
                        taxonomy, references,
                        B155_B156, B173_B174, B013_B014, B133_B134,
                        L111_L112, B167_B168, B197_B198, B007_B008,
                        T195_T196, T174, B031_B032,
                        B005_B006))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")

## NMDS analyses
d_rarefied.bray.nmds <- monoMDS(d_rarefied.bray)

## Extract data scores (https://chrischizinski.github.io/rstats/2014/04/13/vegan-ggplot2/)
samples <- rownames(d_rarefied)
stress <- d_rarefied.bray.nmds$stress
data.scores <- as.data.frame(scores(d_rarefied.bray.nmds))
data.scores$site <- rownames(data.scores)
data.scores$samples <- samples
head(data.scores)

## Add a forest variable
data.scores <- mutate(data.scores, forest = substr(site, 1, 1)) %>% select(-site)
data.scores$forest[data.scores$forest == "B"] <- "Panama (Barro Colorado)"
data.scores$forest[data.scores$forest == "L"] <- "Costa Rica (La Selva)"
data.scores$forest[data.scores$forest == "T"] <- "Ecuador (Tiputini)"
x_min <- min(data.scores$MDS1)
y_max <- max(data.scores$MDS2)
stress_annotation <- paste("stress: ", round(stress, digits = 4), sep = "")
head(data.scores)


## Plot
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, label = samples)) +
    geom_text_repel(alpha = 0.5,
                    size = 2.75,
                    segment.size = 0.25,
                    segment.color = "grey",
                    max.iter = iterations) +
    geom_point(aes(colour = forest), size = 2) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    annotate("text", x = x_min + abs(x_min / 10),
             y = y_max, label = stress_annotation) +
    coord_equal()

output_file <- paste(output, "_MDS.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

## Plot (no sample names)
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, colour = forest)) +
    geom_point(size = 3) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    annotate("text", x = x_min + abs(x_min / 10),
             y = y_max, label = stress_annotation) +
    coord_equal()

output_file <- paste(output, "_MDS_no_labels.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(dendextend)
library(dendextendRcpp)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE, row.names = 1)

## -------------------------------------------------------------------------- ##
## Cleaning
## Samples with less than 10,000 reads
## B199_B200, B175_B176, L111_L112, B197_B198, B145_B146, B167_B168, L183_L184, L131_L132, L097_L098, L181_L182
## Outliers
## B173_B174  # Low diversity, mostly made of one OTU of Chlorophyceae Dunaliella

## Reduce (remove useless columns and low samples)
d <- subset(d,
            select = -c(amplicon, total, chimera, identity,
                        taxonomy, references,               
                        B199_B200, B175_B176, L111_L112, B197_B198,
                        B145_B146, B167_B168, L183_L184, L131_L132,
                        L097_L098, L181_L182,
                        B173_B174))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")

## UPGMA (Unweighted Pair Group Method with Arithmetic Mean)
dendrogram <- hclust(d_rarefied.bray, "average") %>% as.dendrogram()

grepl("^T", labels(dendrogram))
colors <- substr(labels(dendrogram), 1, 1)
colors <- gsub("T", "#7CAE00", colors)
colors <- gsub("B", "#00BFC4", colors)
colors <- gsub("L", "#F8766D", colors)

## Plot and Save
output_file <- paste(output, "_UPGMA_prettier.pdf", sep = "")
pdf(file = output_file, height = 6, width = 14)

dendrogram %>%
    set("leaves_pch", 19) %>%
    set("leaves_cex", 1.1) %>%
    set("leaves_col", colors) %>%
    set("labels_cex", 0.6) %>%
    hang.dendrogram %>%
    plot()

dev.off()

quit(save = "no")

library(vegan)
library(tidyr)
library(dplyr)
library(ggplot2)
library(ggrepel)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.fungi.table"
output <- gsub(".table", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = "\t", header = TRUE)

## -------------------------------------------------------------------------- ##
## Cleaning
## Samples with less than 2,000 reads
## Outliers
## B005_B006  # Low diversity?

d <- subset(d,
            select = -c(OTU, amplicon, total, chimera, identity,
                        taxonomy, references,
                        B155_B156, B173_B174, B013_B014, B133_B134,
                        L111_L112, B167_B168, B197_B198, B007_B008,
                        T195_T196, T174, B031_B032,
                        B005_B006))

## transpose (to get OTUs in columns)
d <- t(d)

## Randomly subsample the table, so all samples have the same number of reads
d_rarefied <- rrarefy(d, min(rowSums(d)))

## Bray Curtis dissimilarity matrix
d_rarefied.bray <- vegdist(d_rarefied, method = "bray")

## NMDS analyses
d_rarefied.bray.nmds <- monoMDS(d_rarefied.bray)

## Extract data scores (https://chrischizinski.github.io/rstats/2014/04/13/vegan-ggplot2/)
samples <- rownames(d_rarefied)
data.scores <- as.data.frame(scores(d_rarefied.bray.nmds))
data.scores$site <- rownames(data.scores)
data.scores$samples <- samples
head(data.scores)

## Add a forest variable
data.scores <- mutate(data.scores, forest = substr(site, 1, 1)) %>% select(-site)
data.scores$forest[data.scores$forest == "B"] <- "Panama (Barro Colorado)"
data.scores$forest[data.scores$forest == "L"] <- "Costa Rica (La Selva)"
data.scores$forest[data.scores$forest == "T"] <- "Ecuador (Tiputini)"
head(data.scores)


## Plot
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, label = samples)) +
    geom_text_repel(alpha = 0.5,
                    size = 2.75,
                    segment.size = 0.25,
                    segment.color = "grey",
                    max.iter = 2e4) +
    geom_point(aes(colour = forest), size = 2) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    coord_equal()

output_file <- paste(output, "_MDS.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

## Plot (no sample names)
ggplot(data = data.scores, aes(x = MDS1, y = MDS2, colour = forest)) +
    geom_point(size = 3) +
    theme_bw(base_size = 16) +
    guides(colour = guide_legend(title = NULL)) +
    theme(legend.justification = c(1,0), legend.position = c(1,0)) +
    coord_equal()

output_file <- paste(output, "_MDS_no_labels.pdf", sep = "")
ggsave(output_file, width = 13, height = 8.5)

quit(save = "no")

# aragorn
cd ~/neotropical_diversity/results/first_155_samples/
python ../../src/rarefaction_by_sample.py neotropical_soil_175_samples.OTU.protists_cleaned.table > neotropical_soil_175_samples.OTU.protists_cleaned_rarefaction_by_sample_by_forest.data

library(dplyr)
library(tidyr)
library(ggplot2)
library(scales)

setwd("~/neotropical_diversity/results/first_155_samples/")
input <- "neotropical_soil_175_samples.OTU.protists_cleaned_rarefaction_by_sample_by_forest.data"
## input <- "tmp.data"
output <- gsub(".data", "", input, fixed = TRUE)

## Load the dataframe
d <- read.table(input, sep = " ", header = FALSE)
colnames(d) <- c("forest", "number_of_samples",
                 "median_OTUs", "min_OTUs", "max_OTUs")
d$number_of_samples <-as.factor(d$number_of_samples)
breaks <- floor(seq(1, 88, length.out = 22))

## Rename forests
levels(d$forest)[levels(d$forest) == "Barro"] <- "Panama (Barro Colorado)"
levels(d$forest)[levels(d$forest) == "LaSelva"] <- "Costa Rica (La Selva)"
levels(d$forest)[levels(d$forest) == "Tiputini"] <- "Ecuador (Tiputini)"

## Plot
ggplot(data = d, aes(x = number_of_samples, y = median_OTUs)) +
    geom_point(shape = 19, size = 2, color = "firebrick") +
    geom_errorbar(aes(ymax = max_OTUs, ymin = min_OTUs), width = 0.2, color = "darkgrey") +
    scale_y_continuous(labels = comma) +
    scale_x_discrete(breaks = breaks, labels = breaks) +
    theme_bw(base_size = 16) +
    xlab("number of samples") +
    ylab("number of OTUs") +
    facet_grid(forest ~ .)

output_file <- paste(output, "_median.pdf", sep = "")
ggsave(output_file, width = 10, height = 12)

quit(save = "no")

# kl
cd ${HOME}/neotropical_diversity/data/
PROTISTS="neotropical_soil_175_samples.OTU.protists.table"
grep "Ciliophora" "${PROTISTS}" |\
      awk '{c += 1 ; s += $157} END {print c, s}'
grep "Colpodea" "${PROTISTS}" |\
      awk '{c += 1 ; s += $157} END {print c, s}'

kl
cd ${HOME}/neotropical_diversity/data/
PROTISTS="neotropical_soil_175_samples.OTU.protists.table"
grep "Colpodea" "${PROTISTS}" | cut -f 157,160 | tr "|" "\t" | cut -f 1,6 | \
    awk '{a[$2] += $1 ; b[$2] += 1} END {for (i in a) {print i, a[i], b[i]}}' | \
    sort -k2,2nr

kl
cd ${HOME}/neotropical_diversity/data/
STATS="neotropical_soil_175_samples_1f.stats"
grep -F -f <(grep "Colpodea" "${PROTISTS}" | cut -f 2) "${STATS}" | \
awk '{s += $1} END {print s}'

kl
cd ${HOME}/neotropical_diversity/data/

OTUs="neotropical_soil_175_samples.OTU.protists.table"
OTU_REPRESENTATIVES="${OTUs/.OTU.protists.table/_1f_representatives.fas}"
PROTIST_OTU_REPRESENTATIVES="${OTU_REPRESENTATIVES/_1f_representatives/_protist_OTU}"
PROTIST_OTU_ALL_AMPLICONS="${OTU_REPRESENTATIVES/_1f_representatives/_protist_amplicons}"
SWARMS="${OTU_REPRESENTATIVES/_representatives.fas/.swarms}"
ALL_FASTA="${OTUs/.OTU.protists.table/.fas}"
AMPLICON_TABLE="neotropical_soil_175_samples.amplicons.table"
PROTIST_AMPLICONS_FASTA="${AMPLICON_TABLE/.amplicons.table/_protist_amplicons.fas}"
PROTIST_AMPLICONS_TABLE="${AMPLICON_TABLE/.amplicons.table/_protist_amplicons.table}"

export LC_ALL=C

# Make a new fasta file (representatives)
SEEDS=$(mktemp)
tail -n +2 "${OTUs}" | cut -f 2 > "${SEEDS}"
grep -A 1 -F -f "${SEEDS}" "${OTU_REPRESENTATIVES}" | \
    sed -e '/^--$/d' > "${PROTIST_OTU_REPRESENTATIVES}"

# Make a new fasta file (all amplicons)
PROTISTS=$(mktemp)
grep -F -f "${SEEDS}" "${SWARMS}" | \
    sed -e '/^--$/d' | tr " " "\n" > "${PROTISTS}"
grep -A 1 -F -f "${PROTISTS}" "${ALL_FASTA}" | \
    sed -e '/^--$/d' > "${PROTIST_OTU_ALL_AMPLICONS}"
rm -f "${PROTISTS}" "${SEEDS}"

# Provide an amplicon2sample mapping
AMPLICONS=$(mktemp)
grep "^>" "${PROTIST_AMPLICONS_FASTA}" | \
    tr -d ">" | cut -d "_" -f 1 > "${AMPLICONS}"
grep -F -f "${AMPLICONS}" "${AMPLICON_TABLE}" | \
    sed -e '/^--$/d' > "${PROTIST_AMPLICONS_TABLE}"
rm "${AMPLICONS}"

# Compress and publish
tar cvfj neotropical_175_samples.tar.bz2 "${PROTIST_OTU_REPRESENTATIVES}" "${PROTIST_OTU_ALL_AMPLICONS}" "${PROTIST_AMPLICONS_TABLE}" "${SWARMS}" "${OTUs}"
cp neotropical_175_samples.tar.bz2 /scratch/WWW/

kl
cd ${HOME}/neotropical_diversity/data/
awk -F "_" '/^>/ {s += $2} END {print s}' neotropical_soil_175_samples_protist_amplicons.fas
tail -n +2 neotropical_soil_175_samples.OTU.protists.table | cut -f 157 | awk '{s += $1} END {print s}'

# Amplicon table
wc -l neotropical_soil_175_samples_protist_amplicons.table
grep -c "^>" neotropical_soil_175_samples_protist_amplicons.fas

kl
cd ${HOME}/neotropical_diversity/data/
awk '{if (NR == 1) {
         for (i=2 ; i<NF; i++) {
             n[i] = $i
         }
      } else {
         for (i=2 ; i<NF; i++) {
             s[i] += $i
         }
      }
     } END {
         OFS = "\t"
         for (i=2; i<NF ; i++) {
             print i - 1, n[i], s[i]
         }
     }' neotropical_soil_175_samples.amplicons.table