---
title: "Taxonomic Backbone - sPlot 3.0"
author: "Francesco Maria Sabatini"
date: "`r format(Sys.time(), '%d %B, %Y')`"
tags: 
- database
- big data
- traits
- taxonomy
output: 
  html_document:
    number_sections: true
    toc: true
    toc_depth: 2
abstract: "This document describes the workflow (with contributions from Oliver Purschke, Jürgen Dengler and Florian Jansen) that was used to generate the taxonomic backbone that standardizes taxon names across the (i) global vegetation plot database sPlot version 3.0 and (ii) the global plant trait data base TRY version 5.  "
urlcolor: blue
---


<center>
  ![](/data/sPlot/users/Francesco/_sPlot_Management/splot-long-rgb.png "sPlot Logo")
</center>


***

**Timestamp:** `r date()`  
**Drafted:** Francesco Maria Sabatini  
**Revised:** Helge Bruelheide, Borja Jimenez-Alfaro  
**Version:** 1.1  

**Changes to Version 1.1** Additional manual cleaning of species names from BJA, UJ and HB.  

***

# Data preparation
## Load packages 

```{r results="hide", message=F, warning=F}
library(reshape2)
library(tidyverse)
library(readr)
library(data.table)
library(knitr)
library(kableExtra)
library(stringr)
library(taxize)
library(Taxonstand)
library(vegdata)
```

## Read in taxon names from [sPlot](https://www.idiv.de/sdiv/working_groups/wg_pool/splot/splot_database.html) and [TRY](https://www.try-db.org/TryWeb/Home.php)


```{r, echo=F}
## fungi genera #NOT COMPLETE LIST
mushroom <- c("Mycena", "Boletus", "Russula","Calocybe","Collybia","Amanita","Amanitopsis","Coprinus",
  "Galerina","Geoglossum","Hebeloma","Hydnum","Lactarius","Leucocarpia","Naucoria","Otidea","Polyporus", "Involucrothele",
  "Sarcodom","Sarcoscyphus","Scleroderma","Stropharia","Tylopilus","Typhula", "Calyptella", "Chrysopsora", "Lacrymaria", "Dermoloma", 
   "Agaricus","Alnicola", "Amanitina", "Bovista", "Cheilymenia","Clavulinopsis", "Clitocybe", "Entoloma", "Geaster", "Inocybe",
  "Laccaria", "Laetiporus", "Lepista", "Macrolepiota", "Macrolepis", "Marasmius", "Panaeolus", "Psathyrella", "Psilocybe", 
  "Rickenella", "Sarcoscypha", "Vascellum", "Ramaria", 
  "Amphoroblasia", "Amphoroblastia")
```


```{r, eval=F}
#import and save splot names from DT table
DT0 <- readr::read_delim("../sPlot_data_export/sPlot_3_0_2_species.csv", 
                            delim="\t", 
                         col_type = cols(
                                PlotObservationID = col_double(),
                                Taxonomy = col_character(),
                                `Taxon group` = col_character(),
                                `Taxon group ID` = col_double(),
                                `Turboveg2 concept` = col_character(),
                                `Matched concept` = col_character(),
                                Match = col_double(),
                                Layer = col_double(),
                                `Cover %` = col_double(),
                                `Cover code` = col_character(),
                                x_ = col_double()
                              )
                         ) 
## Exclude fungi
splot.species <- DT0 %>%
  rename(Species.original=`Turboveg2 concept`, Matched.concept=`Matched concept`) %>%
  filter(`Taxon group`!="Mushroom") %>%
  dplyr::select(Species.original, Matched.concept) %>%
  distinct() %>%
  group_by(Matched.concept) %>%
  mutate(fungi= word(Matched.concept, 1) %in% mushroom) %>%
  ungroup() %>%
  filter(fungi==F) %>%
  dplyr::select(Species.original, Matched.concept)

  
write_csv(splot.species, path = "../_derived/splot3.0.2.species.csv")
```

!!! I used the column from TRY with the full species name, not the column with only a two-word name strings

```{r, message=F}
splot.species <- read_csv("../_derived/splot3.0.2.species.csv")

try.species <- readr::read_csv("../_input/AccSpecies_TRY5.csv", col_names = F, locale = locale(encoding = 'Latin1')) %>%
  dplyr::select(-X6, -X7) %>%
  rename(try.ID=X1, FullSpecies=X2, Species=X3, Genus=X4, Family=X5, GrowthForm=X8)

# Sneak in  species from the Alpine database (Borja & Riccardo), as a courtesy to Project #18
alpine.species <- read_delim("../_input/new_alpine_species.txt", col_names = F, delim = "\t", locale = locale(encoding = 'Latin1')) %>% 
  rename(Species=X1)
```
Use the `Matched.concept` column, as it already contains some standardization by [Stephan Hennekkens](https://www.wur.nl/de/Personen/drs.-SM-Stephan-Hennekens.htm) according to [synbiosys](https://www.synbiosys.alterra.nl/turboveg/contact.htm).  
  
sPlot 3.0.1 contains `r nrow(unique(splot.species[,2]))` different species names.  
TRY 5. contains `r nrow(try.species)`.  
I add to this a list of `r nrow(alpine.species)` alpine species delivered from Riccardo Testolin, within sPlot Project #18.

## Combine species lists

```{r, message=F}
spec.list.TRY.sPlot <- splot.species %>%
  dplyr::select(Matched.concept) %>%
  rename(Species=Matched.concept) %>%
  mutate(Source="S") %>%
  bind_rows(try.species %>% 
              dplyr::select(FullSpecies) %>% ##using the full name from TRY
              rename(Species=FullSpecies) %>% 
              mutate(Source="T")) %>%
  bind_rows(alpine.species %>% 
              mutate(Source="A")) %>%
  reshape2::dcast(Species ~ Source) %>%
  mutate(A=ifelse(A>=1, "A", "")) %>%
  mutate(S=ifelse(S>=1, "S", "")) %>%
  mutate(T=ifelse(T>=1, "T", "")) %>%
  mutate(Source=paste(S, T, A, sep="")) %>%
  dplyr::select(-A, -S, -T)
            
 #Number of species unique and in common across databases
```
The *total number of species* in the backbone is `r nrow(spec.list.TRY.sPlot)`.

```{r echo=F}
knitr::kable(spec.list.TRY.sPlot %>%
               mutate(Source=factor(Source, 
                                    levels=c("S", "T", "A", "ST", "SA", "TA", "STA"),
                                    labels=c("sPlot only", "TRY only", "Alpine only",
                                             "sPlot + TRY", "sPlot + Alpine", "TRY + Alpine",
                                             "sPlot + TRY + Alpine"))) %>% 
               group_by(Source) %>%
               summarize(Num.taxa=n()), 
  caption="Number of taxa per database") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")

```

## A-priori cleaning of names
Stripping unwanted characters as well as abbreviation (such as hybrid markers) which would prevent name matching:

```{r}
spec.list.TRY.sPlot <- spec.list.TRY.sPlot %>%
  mutate(OriginalNames=Species) %>%
  dplyr::select(OriginalNames, Species, Source) %>%
  mutate(Species=gsub('*', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('cf. ', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('Cf. ', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('[', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub(']', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub(' x ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('Ã—', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('aff ', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('(', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub(')', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub(' cf ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub(' aff. ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('c‚e', 'ceae', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('    ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('   ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('  ', ' ', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('x-', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('X-', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('×', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('like ', '', Species, fixed=TRUE)) %>% 
  mutate(Species=gsub(',', '', Species, fixed=TRUE)) %>%
  mutate(Species=gsub('_', ' ', Species))
```

For all names, that have a number in their first word, and consist of $>$ 1 words, remove that word:


```{r}
spec.list.TRY.sPlot <- spec.list.TRY.sPlot %>% 
  mutate(firstWordWithNumbers=grepl('[0-9]', word(Species, 1))) %>%
  mutate(numberOfWords= sapply(gregexpr("\\W+", Species), length) + 1) %>%
  mutate(Species=ifelse((firstWordWithNumbers & numberOfWords > 1), 
                        sapply(Species, 
                               function(x) substr(x, start=regexpr(pattern =' ', text=x)+1,
                                                  stop=nchar(x))), Species))
```

Correct some name abbreviations using `taxname.abbr` in `vegdata`:
```{r}
spec.list.TRY.sPlot <- spec.list.TRY.sPlot %>% 
  mutate(Species=taxname.abbr(spec.list.TRY.sPlot$Species)) %>%
  dplyr::select(OriginalNames, Species, Source)  %>% 
  distinct()
```

A total of `r nrow(spec.list.TRY.sPlot %>% filter(OriginalNames != Species))` species names were modified. Although substantially improved, the species list has still quite a lot of inconsistencies.
The total list submitted to TNRS contains `r length(unique(spec.list.TRY.sPlot$Species))` species names.


# Match names against Taxonomic Name Resolution Service ([TNRS](http://tnrs.iplantcollaborative.org))
Export species name list
```{r }
write_csv(spec.list.TRY.sPlot %>% dplyr::select(Species) %>% distinct() ,  
          path = "../_derived/TNRS_submit/tnrs_submit_iter1.csv")
```

The csv-file of species names was submitted to Taxonomic Name Resolution Service web application (Boyle et al. 2013, iPlant Collaborative (2015)). TNRS version 4.0 was used, which became available in August 2015 (this version also included The Plant List version 1.1).
TNRS was queried on 27/07/2019.


## TNRS settings {#ID}
The following settings were used for resolving names on TNRS.

### Sources for name resolution {#ID}
The initial TNRS name resolution run was based on the **five standard sources** that were **ranked according to preference** in the following order (default of TNRS):

1. The Plant List ([TPL](http://www.theplantlist.org/))[@TPL2013]
2. The Global Compositae Checklist ([GCC](http://compositae.landcareresearch.co.nz/))[@Flann2009]
3. The International Legume Database and Information Service ([ILDIS](http://www.ildis.org/))[@ILDIS2006]
4. [Tropicos](http://www.tropicos.org/) [@TROPICOS2013]
5. PLANTS Database ([USDA](https://plants.usda.gov/java/))[@USDA2012]


### Family Classification
Resolved names were assigned to families based on the [APGIII classification](http://onlinelibrary.wiley.com/doi/10.1111/j.1095-8339.2009.00996.x/abstract) [@Chase2009], the same classification system used by Tropicos.

### Retrieve results
Once the  matching process was finished, results were retrieved from TNRS using the `Detailed Download` option that included the full name information (parsed components, warnings, links to sources, etc.).  We retrieved all the matches for each species, constrained by source (TNRS default), where the name in the first source was selected as best match, unless there was `no suitable match found` in that source, the match from the next lower-ranked source was selected, until all resources where exhausted.

### General procedure  {#ID}
Manually inspect the TNRS-results table in a spreadsheat application (i.e. LibreOffice or Excel). Starting with the highest taxonomic rank considered (i.e. Family). For instance, if manual checking of the TRNS output reveals that all accepted names or synonyms that have accuracy scores >0.9 are correct taxon names, use the following selection procedure:

* Name_matched_rank (==Family)
* Taxonomic_status (==Accepted, Synomyn)
* Family_score (>0.9)

Continue this selection procedure for entries that were matched at lower taxonomic ranks, i.e. genus, species, etc..

## Iteration 1 - Read and combine TNRS result files

Read the files downloaded from TNRS  into `R`.
```{r }
tnrs.res0 <- readr::read_delim("../_derived/TNRS_submit/tnrs_results_iter1.txt", delim="\t", locale = locale(encoding = 'UTF-8'),quote="",
          col_type = cols(
                .default = col_character(),
                Name_number = col_double(),
                Overall_score = col_double(),
                Name_score = col_double(),
                Author_score = col_double(),
                Family_score = col_double(),
                Genus_score = col_double(),
                Specific_epithet_score = col_double(),
                Infraspecific_epithet_score = col_double(),
                Infraspecific_epithet_2_score = col_double(),
                Selected = col_logical()
              ))
```


### Select best match for each submitted name

Best matches are selected in successive steps, depending at which taxonomic level each record was matched. Records were sorted based on decreasing match scores. Matches at low taxonomic level (variety, subspecies) were favoured over matches at high taxonomic levels (family, sections). When having exactly the same ranks, the records were ranked based on their source, as explained above.  
For each name submitted, only the record having the highest rank was retained.


```{r}
tnrs.res <- tnrs.res0 %>%
  mutate(Name_matched_rank=factor(Name_matched_rank, 
                                  levels=c("variety", "subspecies", "species", "genus",
                                           "family", "section", "supersection",
                                           "infraspecies", "forma", "race",
                                           "nothosubspecies", "proles", "monstr",
                                           "series"))) %>%
  mutate(Source=factor(Source, levels=c("tpl",    #reorder priorities
                                        "tpl;gcc",
                                        "tpl;gcc;tropicos",
                                        "tpl;gcc;tropicos;usda",
                                        "tpl;gcc;usda",
                                        "tpl;ildis",
                                        "tpl;ildis;tropicos",
                                        "tpl;ildis;usda",
                                        "tpl;tropicos",
                                        "tpl;tropicos;usda",
                                        "tpl;usda",
                                        "gcc",
                                        "gcc;tropicos",
                                        "gcc;tropicos;usda",
                                        "gcc;usda", 
                                        "ildis", 
                                        "ildis;tropicos",
                                        "ildis;tropicos;usda",
                                        "ildis;usda",
                                        "tropicos",
                                        "tropicos;gcc",
                                        "tropicos;usda",
                                        "usda"  ))) %>%
  mutate(Taxonomic_status=factor(Taxonomic_status, 
                                levels=c("Accepted","Synonym", "No opinion","Invalid","Illegitimate","Misapplied","Rejected name"))) %>%
  #filter(Taxonomic_status %in% c("Accepted", "Synonym")) %>%
  arrange(Name_number, 
          desc(Infraspecific_epithet_2_score),
          desc(Infraspecific_epithet_score), 
          desc(Specific_epithet_score),
          desc(Genus_score), 
          desc(Family_score),
          desc(Name_score),
          desc(Overall_score), 
          #Taxonomic_status,
          Source) %>%
  group_by(Name_submitted) %>%
  slice(1)
```

After this first step, there are `r sum(tnrs.res$Name_matched=="No suitable matches found.")` records for which no match was found. Another `r sum(tnrs.res$Overall_score<0.9)` were unreliably matched (overall match score <0.9). 


### Family level {#ID}
Manually inspect sorted table and select all entries at the highest hierarchical level (family). Manually identify the family accuracy score threshold value above which a name can be considered a correct name. In the following case, this corresponds to a score $>$0.88.

```{r}
index.family <- which(tnrs.res$Name_matched_rank == "family" &
                               (tnrs.res$Taxonomic_status == "Accepted" |
                                tnrs.res$Taxonomic_status == "Synonym") &
                                tnrs.res$Family_score > 0.88)
length(index.family)
```


### Genus level
```{r}
index.genus <- which(tnrs.res$Name_matched_rank == "genus" &
                        (       tnrs.res$Taxonomic_status %in% c("Synonym", "Accepted") &
                                tnrs.res$Genus_score > 0.83) 
                     |
                        (       tnrs.res$Taxonomic_status == "No opinion" &
                                tnrs.res$Genus_score >= 0.99))
length(index.genus)
```
### Species level
```{r }
index.species  <- which(tnrs.res$Name_matched_rank == "species" &
                            (     (tnrs.res$Taxonomic_status == "Accepted" |  #condition 1
                                  tnrs.res$Taxonomic_status == "Synonym") &
                                  tnrs.res$Genus_score > 0.78 &
                                  tnrs.res$Name_score > 0.90) 
                        |
                            (     tnrs.res$Genus_score > 0.90 &         # condition 2 - effective for records with subspecies information
                                  (tnrs.res$Specific_epithet_score > 0.90) 
                        ))
length(index.species)
```

### Subspecies level
```{r, eval = T}
index.subspec <- which( (tnrs.res$Name_matched_rank %in% c("infraspecies", "subspecies") |
                                   is.na(tnrs.res$Name_matched_rank)) & # there are a few records at sub-species level which are not categorized
                                (tnrs.res$Taxonomic_status == "Accepted" |
                                 tnrs.res$Taxonomic_status == "Synonym"))
length(index.subspec)
index.variety <- which(tnrs.res$Name_matched_rank == "variety" &
                                (tnrs.res$Taxonomic_status == "Accepted" |
                                 tnrs.res$Taxonomic_status == "Synonym"))
length(index.variety)

index.infraspec <- which(tnrs.res$Name_matched_rank == "infraspecies")
length(index.infraspec)

index.forma <- which(tnrs.res$Name_matched_rank == "forma")
length(index.forma)
```

### Identifying "non-matched" species that are spermatophyta
```{r, eval = T}
index.spermatophyt <- which(tnrs.res$Name_matched == "No suitable matches found."
                                     & word(tnrs.res$Name_submitted, 1) == "Spermatophyta")
length(index.spermatophyt)
```

### Select `certain` or `uncertain` names
Select names that do not fulfill the search criteria, i.e. that were not selected as certain species, for further name matching.

```{r, eval = T}
index.tnrs <- unique(c(index.family, index.forma, index.genus, index.species, index.subspec,
               index.variety, index.spermatophyt))

tnrs.res.certain <- tnrs.res[index.tnrs,]
dim(tnrs.res.certain)
write.csv(tnrs.res.certain, file = "../_derived/TNRS_submit/tnrs.res.iter1.certain.csv")

tnrs.res.uncertain <- tnrs.res[-index.tnrs,]  
dim(tnrs.res.uncertain)
write.csv(tnrs.res.uncertain, file = "../_derived/TNRS_submit/tnrs.res.iter1.uncertain.csv")

save(tnrs.res.certain, tnrs.res.uncertain, file="../_derived/TNRS_submit/tnrs.iter1.RData")

```

### Manual cleaning, delete subspecies information and rerun match in TNRS
Many unmatched records do contain subspecies information which could not be retrieved in TNRS, although genus and species seem to be spelled correctly. Also, sometimes the mismatch derives from having the word 'species' or 'sp' at the end of the name. 
```{r}
#Ancillary function to change to lower case
firstup <- function(x) {
  substr(x, 1, 1) <- toupper(substr(x, 1, 1))
  x
}
#Manual cleaning
tnrs.submit.iter2 <- data.frame(old=tnrs.res.uncertain$Name_submitted) %>%
  mutate(new=old) %>%
  mutate(new=tolower(new)) %>%
  mutate(new=firstup(new)) %>%
  mutate(new=gsub("^Str ", "", new)) %>%
  mutate(new=gsub("^Unknown ", "", new)) %>%
  mutate(new=firstup(new)) %>%
  mutate(new=gsub(" [0-9]*$", "", new)) %>%  #delete digits at end of object
  mutate(new=gsub("^\\d+|\\d+$", "", new)) %>% #delete digits at the beginning or end of a string
  mutate(new=gsub(" sp.$", "", new)) %>%
  mutate(new=gsub(" sp$", "", new)) %>%
  mutate(new=gsub(" species$", "", new)) %>%
  mutate(new=gsub(" *$", "", new)) %>%
  mutate(new=gsub(" #$", "", new)) %>%
  mutate(new=gsub(" m$", "", new)) %>%
  mutate(new=gsub("acea ", "aceae ", new)) %>%
  mutate(new=gsub('^Agropyrum', 'Agropyron', new)) %>%
  mutate(new=gsub('^Anno ', 'Annona ', new)) %>%
  mutate(new=gsub('Adpdytes dimidiata', 'Apodytes dimidiata', new)) %>%
  mutate(new=gsub('Adenostorna fasciculaturn', 'Adenostoma fasciculatum', new)) %>%
  mutate(new=gsub('Arctostapliylos glallca', 'Arctostaphylos glauca', new)) %>%
  mutate(new=gsub('Bituminosa bituminosa', 'Bituminaria bituminosa', new)) %>%
  mutate(new=gsub('Causurina equisitifolia', 'Causuarina equisetifolia', new)) %>%
  mutate(new=gsub('Convulvus arvensis', 'Convolvulus arvensis', new)) %>%
  mutate(new=gsub('Diospyrus dygina', 'Diospyros dygina', new)) %>%
  mutate(new=gsub('^Dodoea', 'Dodonaea', new)) %>%
  mutate(new=gsub('^Boheravia', 'Boerhavia', new)) %>%
  mutate(new=gsub('Centaria maculosa', 'Centaurea maculosa', new)) %>%
  mutate(new=gsub('Chamrenerium angustifolium', 'Chamaenerion angustifolium', new)) %>%
  mutate(new=gsub('^Chicorium', 'Cichorium', new)) %>%
  mutate(new=gsub('^Cirsiumum', 'Cirsium', new)) %>%
  mutate(new=gsub('^Colubrium', 'Colubrina', new)) %>%
  mutate(new=gsub('^Corymbium', 'Corymbia', new)) %>%
  mutate(new=gsub('Cosmos bipinnata', 'Cosmos bipinnatus', new)) %>%
  mutate(new=gsub('Diospyrus dygina', 'Diospyros digyna', new)) %>%
  mutate(new=gsub('Diospyros egbert', 'Diospyros egbert-walkeri', new)) %>%
  mutate(new=gsub('Dispyrus halesioides', 'Diospyros halesioides', new)) %>%
  mutate(new=gsub('^Drymis', 'Drimys', new)) %>%
  mutate(new=gsub('^Dysoxylon', 'Dysoxylum', new)) %>%
  mutate(new=gsub('^Eleaegnus', 'Elaeagnus', new)) %>%
  mutate(new=gsub('^Eleutherant', 'Eleutherantera', new)) %>%
  mutate(new=gsub('^Echicea', 'Echinacea', new)) %>%
  mutate(new=gsub('Gauteria foliolata', 'Gaultheria foliolosa', new)) %>%
  mutate(new=gsub('^Geophylla', 'Geophyla', new)) %>%
  mutate(new=gsub('Gloichidion insignis', 'Glochidion insigne', new)) %>%
  mutate(new=gsub('^Glycium', 'Glycine', new)) %>%
  mutate(new=gsub('^Hammalis', 'Hamamelis', new)) %>%
  mutate(new=gsub('^Hippochoeris', 'Hypochaeris', new)) %>%
  mutate(new=gsub('Ilix tephrohylla', 'Ilex tephrophylla', new)) %>%
  mutate(new=gsub('^Jasininum', 'Jasminum', new)) %>%
  mutate(new=gsub('Jenipa conjuta', 'Jenipa conjunta', new)) %>%
  mutate(new=gsub('^Lechytis', 'Lecythis', new)) %>%
  mutate(new=gsub('Lespedeza juncus', 'Lespedeza juncea', new)) %>%
  mutate(new=gsub('Licania apelata', 'Licania apetala', new)) %>%
  mutate(new=gsub('Limeum arenicola', 'Limeum arenicolum', new)) %>%
  mutate(new=gsub('^Maniota', 'Manihot', new)) %>%
  mutate(new=gsub('^Menta', 'Mentha', new)) %>%
  mutate(new=gsub('Metophyum brownei', 'Metopium brownei', new)) %>%
  mutate(new=gsub('Miliusa tomentosum', 'Miliusa tomentosa', new)) %>%
  mutate(new=gsub('Mimululus ringens', 'Mimulus ringens', new)) %>%
  mutate(new=gsub('Nardus strictus', 'Nardus stricta', new)) %>%
  mutate(new=gsub('Neea glomeratha', 'Neea glomerata', new)) %>%
  mutate(new=gsub('^Onopordon', 'Onopordum', new)) %>%
  mutate(new=gsub('^Orbigynia', 'Orbignya', new)) %>%
  mutate(new=gsub('Orites excelsa', 'Orites excelsus', new)) %>%
  mutate(new=gsub('Paedorata lutea', 'Paederota lutea', new)) %>%
  mutate(new=gsub('Palaquin ellipticum', 'Palaquium ellipticum', new)) %>%
  mutate(new=gsub('Palmeria arfakensis', 'Palmeria arfakiana', new)) %>%
  mutate(new=gsub('Petalostcmum purpureum', 'Petalostemum purpureum', new)) %>%
  mutate(new=gsub('Petalostimum purpureum', 'Petalostemum purpureum', new)) %>%
  mutate(new=gsub('^Petrosileum', 'Petroselinum', new)) %>%
  mutate(new=gsub('Phlomis herba', 'Phlomis herba-venti', new)) %>%
  mutate(new=gsub('^Phyllirea', 'Phillyrea', new)) %>%
  mutate(new=gsub('Physilus pumula', 'Physalus pumila', new)) %>%
  mutate(new=gsub('Picea maria', 'Picea mariana', new)) %>%
  mutate(new=gsub('Picea retroXexa', 'Picea retroflexa', new)) %>%
  mutate(new=gsub('Pilayella litoralis', 'Pilayella littoralis', new)) %>%
  mutate(new=gsub('Placocarpus schaereri', 'Platecarpus schaerer', new)) %>%
  mutate(new=gsub('Placocarpus schraereri', 'Platecarpus schaerer', new)) %>%
  mutate(new=gsub('^Pulteea', 'Pultenaea', new)) %>%
  mutate(new=gsub('Quercus rubrum', 'Quercus rubra', new)) %>%
  mutate(new=gsub('Rubus fruticosa', 'Rubus fruticosus', new)) %>%
  mutate(new=gsub('Rubus saxatile', 'Rubus saxatilis', new)) %>%
  mutate(new=gsub('Rubus sylvatici', 'Rubus sylvaticus', new)) %>%
  mutate(new=gsub('^Sanguiria', 'Sanguinaria', new)) %>%
  mutate(new=gsub('Sarauja nepaulensis', 'Sarauja nepalensis', new)) %>%
  mutate(new=gsub('^Sateria', 'Setaria', new)) %>%
  mutate(new=gsub('Sauraiea nepulensis', 'Saurauia nepalensis', new)) %>%
  mutate(new=gsub('Schneckia australis', 'Schenckia australis', new)) %>%
  mutate(new=gsub('Smirnium oleastrum', 'Smyrnium olusatrum', new)) %>%
  mutate(new=gsub('Solms laubachia', 'Solms-laubachia himalayensis', new)) %>%
  mutate(new=gsub('Stellaria chamaejasme', 'Stellera chamaejasme', new)) %>%
  mutate(new=gsub('Steraria parviflora', 'Setaria parviflora', new)) %>%
  mutate(new=gsub('^Stuartia', 'Stewartia', new)) %>%
  mutate(new=gsub('Sycops sinensis', 'Sycopsis sinensis', new)) %>%
  mutate(new=gsub('Tacetum vulgare', 'Tanacetum vulgare', new)) %>%
  mutate(new=gsub('Talinurn angustissimun', 'Talinun angustissimun', new)) %>%
  mutate(new=gsub('Talloma hodgsoni', 'Talauma hodgsonii', new)) %>%
  mutate(new=gsub('Taraxacum albo', 'Taraxacum album', new)) %>%
  mutate(new=gsub('Tetragonia falcata', 'Tetragona falcata', new)) %>%
  mutate(new=gsub('Trapogogon', 'Tragopogon', new)) %>%
  mutate(new=gsub('Zyzyphus saeri', 'Zizyphus saeri', new)) %>%
  mutate(new=gsub('^Helicrysum', 'Helichrysum', new)) %>%
  mutate(new=gsub('^Diceropappus rhinocerotis', 'Elytropappus rhinocerotis', new)) %>%
  mutate(new=gsub('^Euphorbiace ', 'Euphorbiacaea ', new)) %>%
  mutate(new=gsub('^Gloecapsa', 'Gloeocapsa', new)) %>%
  mutate(new=gsub('Glycirhiza', 'Glycyrrhiza', new)) %>%
  mutate(new=gsub('Abiesnordmannia', 'Abies nordmannia', new)) %>%
  mutate(new=gsub('Alnus inca', 'Alnus incana', new)) %>%
  mutate(new=gsub('Amalencier alnifolia', 'Amalenchier alnifolia', new)) %>% 
  mutate(new=gsub('Antylis barba-jovis', 'Anthyllis barba-jovis', new)) %>% 
  mutate(new=gsub('^Albizzia "', 'Albizia ', new)) %>% 
  mutate(new=gsub('^Ipomoena ', 'Ipomoea ', new)) %>% 
  mutate(new=gsub('^Ipomea ', 'Ipomoea ', new)) %>% 
  mutate(new=gsub('Ipomo wolco', 'Ipomoea wolcottiana', new)) %>% 
  ## additional manual cleaning from UJ, BJA, HB
  mutate(new=gsub('Abacaba palm', 'Oenocarpus balickii', new)) %>% 
  mutate(new=gsub('Acerkuomeii', 'Acer kuomeii', new)) %>% 
  mutate(new=gsub('Adelphacme minima', '', new)) %>% 
  mutate(new=gsub('Alder$', 'Alnus', new)) %>% 
  mutate(new=gsub('Amapa$', 'Tabebuia', new)) %>% 
  mutate(new=gsub('Amapa amargoso', 'Parahancornia amapa', new)) %>% 
  mutate(new=gsub('Amapa doce$', 'Tabebuia', new)) %>% 
  mutate(new=gsub('Amapai$', 'Tabebuia', new)) %>% 
  mutate(new=gsub('Amapaí$', 'Tabebuia', new)) %>% 
  mutate(new=gsub('Amapa m1', 'Tabebuia', new)) %>% 
  mutate(new=gsub('Amaranth$', 'Amaranthus', new)) %>% 
  mutate(new=gsub('Amophora fruticosa', 'Amorpha fruticosa', new)) %>% 
  mutate(new=gsub('Anacardiace ', 'Anacardiaceae ', new)) %>% 
  mutate(new=gsub('Anagallisarvensis', 'Anagallis arvensis', new)) %>% 
  mutate(new=gsub('Anemonenarcissiflora var.', 'Anemone narcissiflora', new)) %>% 
  mutate(new=gsub('Anenome ', 'Anemone', new)) %>% 
  mutate(new=gsub('Anona ', 'Annona ', new)) %>% 
  mutate(new=gsub('Antylis ', 'Anthyllis', new)) %>% 
  mutate(new=gsub('Apocyncadea gelbblueh$', 'Apocynaceae', new)) %>% 
  mutate(new=gsub('Aracium', 'Crepis', new)) %>% 
  mutate(new=gsub('Ardis mexic', 'Ardisia mexicana subsp. siltepecana', new)) %>% 
  mutate(new=gsub('Ardis verap', 'Ardisia verapazensis', new)) %>% 
  mutate(new=gsub('Argenomne hummemannii', 'Argemone hunnemanni', new)) %>% 
  mutate(new=gsub('Artabotus', 'Artabotrys', new)) %>% 
  mutate(new=gsub('Artemisiaintegrifolia', 'Artemisia integrifolia', new)) %>% 
  mutate(new=gsub('Asclepiacea$', 'Asclepiadaceae', new)) %>% 
  mutate(new=gsub('Asclep. klimmer', 'Asclepiadaceae', new)) %>% 
  mutate(new=gsub('Astartoseris triquetra', 'Lactuca triquetra', new)) %>% 
  mutate(new=gsub('Asteracee ', 'Asteraceae ', new)) %>% 
  mutate(new=gsub('Avenula glauc$', 'Avenula', new)) %>% 
  mutate(new=gsub('Baikea plurijuga', 'Baikiaea plurijuga', new)) %>% 
  mutate(new=gsub('Binse rundbl', 'Juncaceae', new)) %>% 
  mutate(new=gsub('Blättrige fabaceae th', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Bonel macro$', 'Bonellia macrocarpa subsp. macrocarpa', new)) %>% 
  mutate(new=gsub('Boraginacee samtig', 'Boraginaceae', new)) %>% 
  mutate(new=gsub('Bri¢fitos', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Bryophyte$', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Bryopsida', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Carallia macrophylla', 'Carallia', new)) %>% 
  mutate(new=gsub('Carexectabilis', 'Carex spectabilis', new)) %>% 
  mutate(new=gsub('Carex fein', 'Carex', new)) %>% 
  mutate(new=gsub('Cerania vermicularis', 'Thamnolia vermicularis', new)) %>% 
  mutate(new=gsub('Chamelauci merredin', 'Chamelaucium', new)) %>% 
  mutate(new=gsub('Chamelau drummon', 'Chamelaucium', new)) %>% 
  mutate(new=gsub('Charophyta', 'Characeae', new)) %>% 
  mutate(new=gsub('Cheiridopsis-keimlinge', 'Cheiridopsis', new)) %>% 
  mutate(new=gsub('Chenopodiacee$', 'Chenopodiaceae', new)) %>% 
  mutate(new=gsub('Chiangioden mexicanum', 'Chiangiodendron mexicanum', new)) %>% 
  mutate(new=gsub('Chiranthode pentadactylon', 'Chiranthodendron pentadactylon', new)) %>% 
  mutate(new=gsub('Chrysobalan ', 'Chrysobalanus ', new)) %>% 
  mutate(new=gsub('Cladapodiella', 'Cladopodiella', new)) %>% 
  mutate(new=gsub('Cleidium ', 'Cleidion ', new)) %>% 
  mutate(new=gsub('Collema/leptogium lichenoides', 'Collemataceae', new)) %>% 
  mutate(new=gsub('Comarostaph discolor', 'Comarostaphylis discolor', new)) %>% 
  mutate(new=gsub('Combretdodendrum africana', 'Combretodendrum africanum', new)) %>% 
  mutate(new=gsub('Commelinacaea floscopa', 'Floscopa glomerata', new)) %>% 
  mutate(new=gsub('Coyncia setigera', 'Coincya setigera', new)) %>% 
  mutate(new=gsub('Crataeva', 'Crateva', new)) %>% 
  mutate(new=gsub('Craterosperma', 'Rubiaceae', new)) %>% 
  mutate(new=gsub('Crespicium', 'Burseraceae', new)) %>% 
  mutate(new=gsub('Critoniadel nubigenus', 'Critoniadelphus nubigenus', new)) %>% 
  mutate(new=gsub('Crotalaria/vigna?', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Croto billb', 'Croton billbergianus subsp. pyramidalis', new)) %>% 
  mutate(new=gsub('Dana„ racemosa', 'Danae racemosa', new)) %>% 
  mutate(new=gsub('Deehasia', 'Dehaasia', new)) %>% 
  mutate(new=gsub('Dichapetala', 'Dichapetalum', new)) %>% 
  mutate(new=gsub('Distel bractea', 'Asteracaea', new)) %>% 
  mutate(new=gsub('Distelig asteraceae', 'Asteracaea', new)) %>% 
  mutate(new=gsub('Dodon visco', 'Dodonaea viscosa', new)) %>% 
  mutate(new=gsub('Doldenbluetler', 'Apiaceae', new)) %>% 
  mutate(new=gsub('Echinosurus capitatus', 'Poaceae', new)) %>% 
  mutate(new=gsub('Einähriges gras$', 'Poaceae', new)) %>% 
  mutate(new=gsub('Einähriges gras von gestern$', 'Poaceae', new)) %>% 
  mutate(new=gsub('Einblütiges rispengras', 'Poaceae', new)) %>% 
  mutate(new=gsub('Eiovaltrichtergrundblatt orchidee', 'Orchidaceae', new)) %>% 
  mutate(new=gsub('Elongata subsp.', 'Pohlia elongata', new)) %>% 
  mutate(new=gsub('Enriquebelt ', 'Enriquebeltrania ', new)) %>% 
  mutate(new=gsub('Entermorpha ', 'Enteromorpha ', new)) %>% 
  mutate(new=gsub('Erodiurn$', 'Erodium', new)) %>% 
  mutate(new=gsub('Euc. chloroclada x camaldulensis', 'Eucalyptus', new)) %>% 
  mutate(new=gsub('Euphorbiacée ipatouduluga gouduatché', 'Euphorbiaceae', new)) %>% 
  mutate(new=gsub('Fabacee kleeblatt stengel schwarzdrüsi', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Fabaceenstrauch wie 132446 f', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Fabaceenstr kleinbltrg', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Fabacee wie lotus f', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Farn', 'Pteridophyta', new)) %>% 
  mutate(new=gsub('Farn cystopteris', 'Cystopteris', new)) %>% 
  mutate(new=gsub('Fern', 'Pteridophyta', new)) %>% 
  mutate(new=replace(new, list=word(new, 1)=="Fingergras", values="Digitaria")) %>% 
    mutate(new=replace(new, list=word(new, 1)=="Fingerhirse", values="Digitaria")) %>% 
  mutate(new=gsub('Gelbe onagraceae', 'Onagraceae', new)) %>% 
  mutate(new=gsub('Gramine', 'Poaceae', new)) %>% 
  mutate(new=gsub('Graminea', 'Poaceae', new)) %>% 
  mutate(new=gsub('Graminia', 'Poaceae', new)) %>% 
  mutate(new=gsub('Grannenquecke', 'Poaceae', new)) %>% 
  mutate(new=replace(new, 
                     list=word(new, 1)=="Gras", 
                     values="Poaceae")) %>% 
  mutate(new=gsub('Gynostachi dicanthus', 'Gymnostachium diacanthus', new)) %>% 
  mutate(new=gsub('Hafer haarkranz', 'Poaceae', new)) %>% 
  mutate(new=gsub('Hapolosiphon', 'Hapalosiphon', new)) %>% 
  mutate(new=gsub('Heliocrysum', 'Helichrysum', new)) %>% 
  mutate(new=replace(new, list=word(new, 1)=="Hepaticae", values="Bryophyta")) %>% 
  mutate(new=gsub('Hepaticas', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Hepatophyta', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Hermerocalis', 'Hemerocallis', new)) %>% 
  mutate(new=replace(new, 
                     list=word(new, 1)=="Hirse", 
                     values="Poaceae")) %>% 
  mutate(new=gsub('Hirte trian', 'Hirtella triandra subsp. media', new)) %>% 
    mutate(new=replace(new, list=word(new, 1)=="Hohlzahn", values="Lamiaceae")) %>%
  mutate(new=gsub('Hondurodend urceolatum', 'Hondurodendron urceolatum', new)) %>% 
  mutate(new=gsub('Hornklee gelb', 'Fabaceae', new)) %>% 
  mutate(new=replace(new, 
                     list=word(new, 1)=="Horstgras",
                     values="Poaceae")) %>% 
  mutate(new=replace(new, 
                     list=word(new, 1)=="Huehnerhirse",
                     values="Digitaria")) %>% 
  mutate(new=gsub('Hydrocoleus lyngbyaceus', 'Hydrocoleum lyngbyaceum', new)) %>% 
  mutate(new=gsub('Hyernima nipensis', 'Hieronyma nipensis', new)) %>% 
  mutate(new=gsub('Hyeronima', 'Hieronyma', new)) %>% 
  mutate(new=gsub('Hypocal angusti', 'Hypocalymma angustifolium', new)) %>% 
  mutate(new=gsub('Hypocalym nambung', 'Hypocalymma', new)) %>% 
  mutate(new=gsub('Hyprium', 'Hypericum', new)) %>% 
  mutate(new=gsub('Igelkolben', 'Sparganium', new)) %>% 
  mutate(new=gsub('Ilexã‚â paraguariensis', 'Ilex', new)) %>% 
  mutate(new=gsub('Ipomea', 'Ipomoea', new)) %>% 
  mutate(new=gsub('Ipomoena', 'Ipomoea', new)) %>% 
  mutate(new=gsub('Jm kürbis stark behaart', 'Cucurbitaceae', new)) %>% 
  mutate(new=gsub('Juncaginacee/triglochin', 'Triglochin', new)) %>% 
  mutate(new=gsub('Juncas', 'Juncus', new)) %>% 
  mutate(new=gsub('Keilblatt cyperus', 'Cyperus', new)) %>% 
  mutate(new=gsub('Khh 3010 polygalacee', 'Polygalaceae', new)) %>% 
  mutate(new=gsub('	Khh 3014 liliacee 3f„ch. kapsel schwarze samen', 'Liliaceae', new)) %>% 
  mutate(new=gsub('Khh 3024 brachiaria', 'Brachiaria', new)) %>% 
  mutate(new=gsub('Khh 3025 liliaceae gelbe blten breite bl„tter', 'Liliaceae', new)) %>% 
  mutate(new=gsub('Khh 3037 ficus', 'Ficus', new)) %>% 
  mutate(new=gsub('Khh 3054 ficus iteophylla miq.', 'Ficus', new)) %>% 
  mutate(new=gsub('Kl. borstgras', 'Poaceae', new)) %>% 
  mutate(new=gsub('Kleine malvaceae', 'Malvaceae', new)) %>% 
  mutate(new=replace(new, 
                     list=word(new, 1)=="Kletter", 
                     values="Asteraceae")) %>% 
  mutate(new=gsub('Klimmer asclepiadaceae', 'Asclepiadaceae', new)) %>% 
  mutate(new=gsub('Klimmer curcuvitaceae', 'Cucurbitaceae', new)) %>% 
  mutate(new=gsub('Kl. sauergras', 'Cyperaceae', new)) %>% 
  mutate(new=gsub('Knabenkraut gefleckt', 'Orchis', new)) %>% 
  mutate(new=gsub('Knubbelblüt. gras haarkranz vgl f', 'Poaceae', new)) %>% 
  mutate(new=replace(new, 
                   list=word(new, 1)=="Koenigskerze", 
                   values="Verbascum")) %>% 
  mutate(new=gsub('Kriechgras zynodon', 'Poaceae', new)) %>% 
  mutate(new=gsub('Kürbis', 'Cucurbitaceae', new)) %>% 
  mutate(new=gsub('Lamiaceen strauch', 'Lamiaceae', new)) %>% 
  mutate(new=gsub('Lamiacee orange', 'Lamiaceae', new)) %>% 
  mutate(new=gsub('Lamiales orobanchaceae + phrymaceae + plantaginaceae + scrophulariaceae', 'Orobanchaceae', new)) %>% 
  mutate(new=gsub('Lantanacamara wandelrösschen', 'Lantana camara', new)) %>% 
  mutate(new=gsub('Lasiopeta watheroo k. shepherd & c. wilkins ks', 'Lasiopetalum', new)) %>% 
  mutate(new=gsub('Leg-inderteminada', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Legu 1fiedrig groá schlank', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Legume$', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Leguminosae spgm', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Leguminosea', 'Fabaceae', new)) %>% 
  mutate(new=replace(new, 
                 list=word(new, 1)=="Leguminose", 
                 values="Fabaceae")) %>% 
  mutate(new=gsub('Leheelo grass', 'Poaceae', new)) %>% 
  mutate(new=gsub('Lepid carra', 'Lepiderema', new)) %>% 
  mutate(new=gsub('Lich caloplaca', 'Caloplaca', new)) %>% 
  mutate(new=gsub('Liliacee', 'Liliaceae', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Lilie", 
               values="Liliaceae")) %>% 
  mutate(new=gsub('Liliengewächs', 'Liliaceae', new)) %>% 
  mutate(new=gsub('Lisea', 'Litsea', new)) %>% 
  mutate(new=gsub('Lisymachia', 'Lysimachia', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Liverwort", 
               values="Bryophyta")) %>% 
  mutate(new=gsub('Livwort', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Lonicerachrysantha', 'Lonicera chrysantha', new)) %>% 
  mutate(new=gsub('Lycoctamnus barbatus', 'Aconitum barbatum', new)) %>% 
  mutate(new=gsub('Lygopus', 'Lycopus', new)) %>% 
  mutate(new=gsub('Maitenus', 'Maytenus', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Malpighiace", 
               values="Malpighiaceae")) %>% 
  mutate(new=gsub('Malpighiales chrysobalanaceae + humiriaceae', 'Malpighiaceae', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Malve", 
               values="Malvaceae")) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Mammutgras", 
               values="Poaceae")) %>% 
  mutate(new=gsub('Mammutgrass', 'Poaceae', new)) %>% 
  mutate(new=gsub('Maqui guian', 'Maquira guianensis subsp. costaricana', new)) %>% 
  mutate(new=gsub('Marchantiophyta', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Mariana aphylla', 'Maireana aphylla', new)) %>% 
  mutate(new=gsub('Mehrfingeriges ährengras', 'Poaceae', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Melastomata", 
               values="Melastomataceae")) %>% 
  mutate(new=gsub('Mesembr minibl', 'Mesembryanthemum', new)) %>% 
  mutate(new=gsub('Mesostomma kotschyanum', 'Mesostemma kotschyana', new)) %>% 
  mutate(new=gsub('Microhepatics', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Micromeria micrantha', 'Micromeria graeca subsp. micrantha', new)) %>% 
  mutate(new=gsub('Mimose minifiedrig f', 'Fabaceae', new)) %>% 
  mutate(new=gsub('Miniepilobium', 'Epilobium', new)) %>% 
  mutate(new=gsub('Minimargerite', 'Asteraceae', new)) %>% 
  mutate(new=gsub('Miniochna', 'Ochna', new)) %>% 
  mutate(new=gsub('Minischilf 132466 f', 'Poaceae', new)) %>% 
  mutate(new=gsub('Mistletoe', 'Viscum', new)) %>% 
  mutate(new=gsub('Mniaecia', 'Mniaceae', new)) %>% 
  mutate(new=gsub('Molemo', 'Turraea', new)) %>% 
  mutate(new=gsub('Molses', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Momisa pigra', 'Mimosa pigra', new)) %>% 
  mutate(new=gsub('Monandrus squarrosus', 'Cyperus squarrosus', new)) %>% 
  mutate(new=gsub('Monchema debile', 'Monechma debile', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Monochna", 
               values="Polygalaceae")) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Moos", 
               values="Bryophyta")) %>% 
  mutate(new=gsub('Moospolster grau-grün', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Mortonioden ', 'Mortoniodendron ', new)) %>% 
  mutate(new=gsub('Mos onbekend', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Mossen overige', 'Bryophyta', new)) %>% 
  mutate(new=gsub('Mougetia', 'Mougeotia', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Musci", 
               values="Bryophyta")) %>% 
  mutate(new=gsub('Myciantes', 'Myrcianthes', new)) %>% 
  mutate(new=gsub('Myrciaã‚â pulchra', 'Myrcia pulchra', new)) %>% 
  mutate(new=gsub('Myrcianov.', 'Myrcia', new, fixed = T)) %>% 
  mutate(new=gsub('Myrsi coria', 'Myrsine coriacea', new)) %>% 
  mutate(new=gsub('Myrtaceenstrauch', 'Myrtaceae', new)) %>% 
  mutate(new=gsub('Nachtkerze fru dreispaltig', 'Onagracaee', new)) %>% 
  mutate(new=gsub('Neobartsia crenoloba', 'Bartsia crenoloba', new)) %>% 
  mutate(new=gsub('None$', 'Nonea', new)) %>%
  mutate(new=gsub('Ocos adenophylla', 'Symplocos adenophylla', new)) %>%
  mutate(new=gsub('Officinale subsp. group', 'Taraxacum officinale s.l.', new)) %>%
  mutate(new=gsub('Orch$', 'Orchidaceae', new)) %>%
  mutate(new=gsub('Orchid', 'Orchidaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Orchidee", 
               values="Orchidaceae")) %>% 
  mutate(new=replace(new, 
               list=word(new, 1) %in% c("Papilonacea", "Papilionacea"),
               values="Fabaceae")) %>% 
  mutate(new=gsub('Pasania dodoniifolia', 'Lithocarpus dodonaeifolius', new)) %>%
  mutate(new=gsub('Phoebengmoensis', 'Phoebe hungmoensis', new)) %>%
  mutate(new=gsub('Picra antid$', 'Picramnia antidesma subsp. fessonia', new)) %>%
  mutate(new=gsub('Pinopsida', 'Coniferae', new)) %>%
  mutate(new=gsub('Pisonianov.', 'Pisonia', new, fixed=T)) %>%
  mutate(new=gsub('Pithecellob ', 'Pithecellobium ', new)) %>%
  mutate(new=gsub('Pithecocten', 'Pithecoctenium', new)) %>%
  mutate(new=gsub('Pleradenoph longicuspis', 'Pleradenophora longicuspis', new)) %>%
  mutate(new=gsub('Pleuranthod ', 'Pleuranthodendron ', new)) %>%
  mutate(new=gsub('Poales', 'Poaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1) %in% c("Polygalacea", "Polygalacee"),
               values="Polygalaceae")) %>%
  mutate(new=replace(new, 
               list=word(new, 1) %in% c("Polygonaceae", "Polygonacee"),
               values="Polygonaceae")) %>% 
  mutate(new=gsub('Polygonumlongisetum', 'Polygonum longisetum', new)) %>%
  mutate(new=gsub('Posoq coria subsp. maxima', 'Posoqueria coriacea subsp. maxima', new)) %>%
  mutate(new=gsub('Prosthecidi ', 'Prosthecidiscus ', new)) %>%
  mutate(new=gsub('Pseudo bidens', '', new)) %>% 
  mutate(new=replace(new, 
               list=word(new, 1) %in% 
                 c("Pseudobriza", "Pseudofingergras", 
                   "Pseudogerste", "Puschelgras", "Quecke",
                   "Queckenblatt", "Queckengras", 
                   "Roggen/hafer", "Ruchgras", "Silbergras", 
                   "Suessgras"), 
               values="Poaceae")) %>% 
  mutate(new=gsub('Ptarmica', 'Achillea', new)) %>%
  mutate(new=gsub('Pterost cauline leaves n. gibson & m.n. lyons', 'Pterostegia', new)) %>%
  mutate(new=gsub('Quararibeaã‚â guianensis', 'Quararibea guianensis', new)) %>%
  mutate(new=gsub('Rainfarn f', 'Asteraceae', new)) %>%
  mutate(new=gsub('Ranke ipomoea', 'Ipomoea', new)) %>%
  mutate(new=gsub('Ranke rubiaceae', 'Rubiaceae', new)) %>%
  mutate(new=gsub('Rauwolfia', 'Rauvolfia', new)) %>%
  mutate(new=gsub('Rheinfarn', 'Asteraceae', new)) %>%
  mutate(new=gsub('Rhodostemon kunthiana', 'Rhodostemonodaphne kunthiana', new)) %>%
  mutate(new=gsub('Riccardia/aneura', 'Bryophyta', new)) %>%
  mutate(new=gsub('Rietgras steril 134051a', 'Poaceae', new)) %>%
  mutate(new=gsub('Rosenbergio formosum', 'Rosenbergiodendron formosum', new)) %>%
  mutate(new=gsub('Rotes puschelgras', 'Poaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Rubiacea", 
               values="Rubiaceae")) %>% 
  mutate(new=gsub('Rytidospe goomallin a.g. gunness et al. oakp 10/', 'Rytidosperma', new)) %>%
  mutate(new=gsub('Salacia idoensis', 'Salacia', new)) %>%
  mutate(new=gsub('Samphire', 'Amaranthaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1) %in% 
                 c("Sauergras", "Schlanksegge", "Sedge", 
                   "Segge", "Simse"),
               values="Cyperaceae")) %>% 
  mutate(new=gsub('Scaev repen subsp. north sandp r.j. cranf & p.j. spenc', 'Scaevola repens', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Schachtelhalm", 
               values="Equisetaceae")) %>% 
  mutate(new=replace(new, 
               list=word(new, 1)=="Schnittlauch", 
               values="Amaryllidaceae")) %>% 
  mutate(new=gsub('Schwertlilie trocken', 'Iridaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1) %in% c("Scropholacea", "Scrophulariacea", "Scroph."),
               values="Scrophulariacea")) %>% 
  mutate(new=gsub('Sitzende onagraceae', 'Onagraceae', new)) %>%
  mutate(new=gsub('Sonnenblume', 'Asteraceae', new)) %>%
  mutate(new=gsub('Stachelgurke', 'Cucurbitaceae', new)) %>%
  mutate(new=gsub('Stark behaarte malve', 'Malvaceae', new)) %>%
  mutate(new=gsub('Staude asteraceae bl watteweich f', 'Asteraceae', new)) %>%
  mutate(new=gsub('Staude crotalaria unterseite silber', 'Crotalaria', new)) %>%
  mutate(new=gsub('Staude solanum', 'Solanaceae', new)) %>%
  mutate(new=gsub('Staude tephrosia', 'Tephrosia', new)) %>%
  mutate(new=gsub('Stipagrosist panicle gross', 'Stipagrostis', new)) %>%
  mutate(new=gsub('Asteraceae u silber', 'Asteraceae', new)) %>%
  mutate(new=gsub('Stratonostoc communeá', 'Stratonostoc commune', new)) %>%
  mutate(new=gsub('Strauch asteraceae nadelblätt.', 'Asteraceae', new)) %>%
  mutate(new=gsub('Strauch blatt wie salix reticulata astera', 'Asteraceae', new)) %>%
  mutate(new=gsub('Strauch blatt wie salix reticulata astera 132534b', 'Asteraceae', new)) %>%
  mutate(new=gsub('Strauch fabaceae gerieft schote', 'Fabaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Strauch" &
                 word(new,2)=="Rubiaceae", 
               values="Rubiaceae")) %>% 
  mutate(new=gsub('Fabaceae samtig bl lanzettlich', 'Fabaceae', new)) %>%
  mutate(new=gsub('Ochna mini', 'Ochna', new)) %>%
  mutate(new=gsub('Stryphnoden microstachyum', 'Stryphnodendron microstachyum', new)) %>%
  mutate(new=gsub('Sumpfgladiole haarig', 'Gladiolus', new)) %>%
  mutate(new=gsub('Sygnum ramphicarpa', 'Scrophulariaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1)=="Symplococar", 
               values="Symplococarpon")) %>% 
  mutate(new=gsub('Sysirinchium', 'Sisyrinchium', new)) %>%
  mutate(new=gsub('Syzigium accuminatisima', 'Syzygium acuminatissimum', new)) %>%
  mutate(new=gsub('Tabernaemon ', 'Tabernaemontana ', new)) %>%
  mutate(new=gsub('Thalassodend', 'Thalassodendron', new)) %>%
  mutate(new=gsub('Thinouia canescens', 'Thinouia', new)) %>%
  mutate(new=gsub('Thistle', 'Asteraceae', new)) %>%
  mutate(new=gsub('Trisetumicatum', 'Trisetum spicatum', new)) %>%
  mutate(new=gsub('Undetermined sedge', 'Cyperaceae', new)) %>%
  mutate(new=replace(new, 
               list=word(new, 1) %in% 
                 c("Liverwort", "Liverworts", "Moss"), 
               values="Bryophyta")) %>% 
  mutate(new=gsub('Vismi bacci', 'Vismia baccifera subsp. ferruginea', new)) %>%
  mutate(new=gsub('Weidenr”schen', 'Onagraceae', new)) %>%
  mutate(new=gsub('Weißpelziger brauner Spross Asteracea', 'Asteraceae', new)) %>%
  mutate(new=gsub('Wie stipagrostis', 'Poaceae', new)) %>%
  mutate(new=gsub('Wincassia', 'Fabaceae', new)) %>%
  mutate(new=gsub('xDactyloden st-quintini', 'Dactylodenia st-quintinii', new)) %>%
  mutate(new=gsub('Zizyphus sp1 IUCN1', 'Zizyphus', new)) %>%
  mutate(new=gsub('Zwiebel Lilaceae steril', 'Lilaceae', new)) %>%
  mutate(new=gsub('Zwstr faurea', 'Faurea', new))
  

# delete remaining records of mushroom species
tnrs.submit.iter2 <- tnrs.submit.iter2 %>%
  filter(!word(new,1) %in% mushroom)
  
# Extract family name for unidentified species
tnrs.submit.iter2 <- tnrs.submit.iter2 %>%
  na.omit() %>%
  group_by(old) %>%
  mutate(family.lev=str_extract(word(new,1), pattern='([^\\s]+aceae)')) %>%
  mutate(new=ifelse(is.na(family.lev), new, family.lev)) %>%
  dplyr::select(-family.lev) %>%
  ungroup()

#Cut to the first 2 words in the name string
tnrs.submit.iter2 <- tnrs.submit.iter2 %>%
  group_by(old) %>%
  mutate(Name_binomial=paste(word(new, c(1,2)), collapse=" ")) %>%
  ungroup() %>%
  mutate(Name_binomial=gsub(' NA$', '', Name_binomial))
```
### Save species list to submit to TNRS for iteration 2
```{r}
write_csv(tnrs.submit.iter2 %>% 
            dplyr::select(Name_binomial) %>% 
            #After cleaning some names now match to those already resolved in iteration 1. Take them out
            filter(!Name_binomial %in% tnrs.res.certain$Name_submitted) %>% 
            distinct(), path="../_derived/TNRS_submit/tnrs_submit_iter2.csv")
```

## Iteration 2 - Reimport resolved species names from TNRS and mark solved

```{r}
tnrs.res.iter2.raw <- readr::read_delim("../_derived/TNRS_submit/tnrs_results_iter2.txt", delim="\t", locale = locale(encoding = 'UTF-8'),quote="",
          col_type = cols(
                .default = col_character(),
                Name_number = col_double(),
                Overall_score = col_double(),
                Name_score = col_double(),
                Author_score = col_double(),
                Family_score = col_double(),
                Genus_score = col_double(),
                Specific_epithet_score = col_double(),
                Infraspecific_epithet_score = col_double(),
                Infraspecific_epithet_2_score = col_double(),
                Selected = col_logical()
              ))

tnrs.res.iter2 <- tnrs.res.iter2.raw %>%
  mutate(Name_matched_rank=factor(Name_matched_rank, 
                                  levels=c("variety", "subspecies", "species", 
                                           "genus", "family", "section", 
                                           "supersection", "infraspecies", "forma", 
                                           "race", "nothosubspecies", "proles", 
                                           "monstr", "series"))) %>%
  mutate(Source=factor(Source, levels=c("tpl",    #reorder priorities
                                        "tpl;gcc", "tpl;gcc;tropicos", "tpl;gcc;tropicos;usda",
                                        "tpl;gcc;usda","tpl;ildis","tpl;ildis;tropicos",
                                        "tpl;ildis;usda","tpl;tropicos","tpl;tropicos;usda",
                                        "tpl;usda","gcc","gcc;tropicos",
                                        "gcc;tropicos;usda","gcc;usda","ildis",
                                        "ildis;tropicos","ildis;tropicos;usda","ildis;usda",
                                        "tropicos","tropicos;gcc","tropicos;usda","usda"  ))) %>%
  mutate(Taxonomic_status=factor(Taxonomic_status, 
                                levels=c("Accepted","Synonym", "No opinion",
                                         "Invalid","Illegitimate","Misapplied",
                                         "Rejected name"))) %>%
 arrange(Name_number, 
          desc(Infraspecific_epithet_2_score),
          desc(Infraspecific_epithet_score), 
          desc(Specific_epithet_score),
          desc(Genus_score), 
          desc(Family_score),
          desc(Name_score),
          desc(Overall_score), 
          Source) %>%
  group_by(Name_submitted) %>%
  slice(1)
```


### Family level
```{r, eval = T}
index.family <- which(tnrs.res.iter2$Name_matched_rank == "family" &
                               (tnrs.res.iter2$Taxonomic_status == "Accepted" |
                                tnrs.res.iter2$Taxonomic_status == "Synonym") &
                                tnrs.res.iter2$Family_score > 0.88)
length(index.family)

```

### Genus level
```{r, eval = T}
index.genus <- which(tnrs.res.iter2$Name_matched_rank == "genus" &
                                 (tnrs.res.iter2$Taxonomic_status %in% c("Accepted","Synonym") &
                                 tnrs.res.iter2$Genus_score >= 0.90 &
                                 tnrs.res.iter2$Name_score > 0.49))
length(index.genus)


```


### Species level
```{r, eval = T}
index.species <- which(tnrs.res.iter2$Name_matched_rank == "species" &
                                 #(tnrs.res.iter2$Taxonomic_status == "Accepted" |
                                 #  tnrs.res.iter2$Taxonomic_status == "Synonym") &
                                  tnrs.res.iter2$Genus_score >= 0.80 &
                                  tnrs.res.iter2$Specific_epithet_score > 0.90)
length(index.species)
```


### Subspecies level
```{r, eval = T}
index.infraspec <- which(tnrs.res.iter2$Name_matched_rank == "infraspecies")
length(index.infraspec)

index.subspec <- which((tnrs.res.iter2$Name_matched_rank %in% c("infraspecies", "subspecies") |
                                   is.na(tnrs.res.iter2$Name_matched_rank)) & # there are a few records at sub-species level which are not categorized
                                (tnrs.res.iter2$Taxonomic_status == "Accepted" |
                                 tnrs.res.iter2$Taxonomic_status == "Synonym"))
length(index.subspec)

index.variety <- which(tnrs.res.iter2$Name_matched_rank == "variety" &
                                (tnrs.res.iter2$Taxonomic_status == "Accepted" |
                                 tnrs.res.iter2$Taxonomic_status == "Synonym"))
length(index.variety)

index.forma <- which(tnrs.res.iter2$Name_matched_rank == "forma")
length(index.forma)

index.spermatophyt <- which(tnrs.res.iter2$Name_matched == "No suitable matches found."
                                     & word(tnrs.res.iter2$Name_submitted, 1) == "Spermatophyta")
length(index.spermatophyt)

```


```{r, eval = T}
index.tnrs.iter2 <- unique(c(index.family, index.forma, index.genus, index.species, index.subspec,
               index.variety, index.spermatophyt))

tnrs.res.iter2.certain <- tnrs.res.iter2[index.tnrs.iter2,]
dim(tnrs.res.iter2.certain)
write.csv(tnrs.res.iter2.certain, file = "../_derived/TNRS_submit/tnrs.res.iter2.certain.csv")

tnrs.res.iter2.uncertain <- tnrs.res.iter2[-index.tnrs.iter2,]
dim(tnrs.res.iter2.uncertain)
write.csv(tnrs.res.iter2.uncertain, file = "../_derived/TNRS_submit/tnrs.res.iter2.uncertain.csv")

save(tnrs.res.iter2.certain, tnrs.res.iter2.uncertain, 
     tnrs.submit.iter2, file="../_derived/TNRS_submit/tnrs.iter2.RData")
```


### Save species list to submit to TNRS for iteration 3
```{r, eval = T}
write_csv(tnrs.res.iter2.uncertain[,2], path = "../_derived/TNRS_submit/tnrs_submit_iter3.csv")
```

This list was submitted to `TNRS`, but only selecting the `NCBI` database.

## Iteration 3 - Reimport resolved species names from `TNRS_NCBI` 


```{r}
tnrs.res.iter3.raw <- readr::read_delim("../_derived/TNRS_submit/tnrs_results_iter3.txt", delim="\t", locale = locale(encoding = 'UTF-8'),quote="",
          col_type = cols(
                .default = col_character(),
                Name_number = col_double(),
                Overall_score = col_double(),
                Name_score = col_double(),
                Author_score = col_double(),
                Family_score = col_double(),
                Genus_score = col_double(),
                Specific_epithet_score = col_double(),
                Infraspecific_epithet_score = col_double(),
                Infraspecific_epithet_2_score = col_double(),
                Selected = col_logical()
              ))

tnrs.ncbi <- tnrs.res.iter3.raw %>%
  mutate(Name_matched_rank=factor(Name_matched_rank, 
                                  levels=c("variety", "subspecies", "species", 
                                           "genus", "family", "section", "supersection",
                                           "infraspecies", "forma", "race",
                                           "nothosubspecies", "proles", "monstr",
                                           "series"))) %>%
  mutate(Source=factor(Source, levels=c("tpl",    #reorder priorities
                                        "tpl;gcc", "tpl;gcc;tropicos", "tpl;gcc;tropicos;usda",
                                        "tpl;gcc;usda","tpl;ildis","tpl;ildis;tropicos",
                                        "tpl;ildis;usda","tpl;tropicos","tpl;tropicos;usda",
                                        "tpl;usda","gcc","gcc;tropicos","gcc;tropicos;usda",
                                        "gcc;usda", "ildis","ildis;tropicos","ildis;tropicos;usda",
                                        "ildis;usda","tropicos","tropicos;gcc","tropicos;usda","usda"  ))) %>%
  mutate(Taxonomic_status=factor(Taxonomic_status, 
                                levels=c("Accepted","Synonym", "No opinion","Invalid","Illegitimate","Misapplied","Rejected name"))) %>%
 arrange(Name_number, 
          desc(Infraspecific_epithet_2_score),
          desc(Infraspecific_epithet_score), 
          desc(Specific_epithet_score),
          desc(Genus_score), 
          desc(Family_score),
          desc(Name_score),
          desc(Overall_score), 
          Source) %>%
  group_by(Name_submitted) %>%
  slice(1)
```

### Family level
```{r, eval = T}
index.family <- which(tnrs.ncbi$Name_matched_rank == "family" &
                                (tnrs.ncbi$Taxonomic_status == "Accepted"|
                                 tnrs.ncbi$Taxonomic_status == "Synonym") &
                                tnrs.ncbi$Family_score > 0.85)
length(index.family)
```
### Genus level
```{r, eval = T}
index.genus <- which(tnrs.ncbi$Name_matched_rank == "genus" &
                       tnrs.ncbi$Taxonomic_status  %in% c("Accepted", "Synonym", "No opinion") &
                         (
                           (tnrs.ncbi$Genus_score > 0.89 & 
                             tnrs.ncbi$Name_score > 0.49)   |
                           (tnrs.ncbi$Genus_score > 0.99 &
                               tnrs.ncbi$Name_score > 0.2)
                          ))
length(index.genus)
```
### Species level
```{r, eval = T}
index.species.1 <- which(tnrs.ncbi$Name_matched_rank == "species" &
                                   (tnrs.ncbi$Taxonomic_status == "Accepted" |
                                    tnrs.ncbi$Taxonomic_status == "Synonym") &
                                    tnrs.ncbi$Name_score > 0.94 & 
                                    tnrs.ncbi$Specific_epithet_score>=0.67)
length(index.species.1)

index.species.2 <- which(tnrs.ncbi$Name_matched_rank == "species" &
                                   (tnrs.ncbi$Taxonomic_status == "Accepted" |
                                   tnrs.ncbi$Taxonomic_status == "Synonym") &
                                   tnrs.ncbi$Genus_score > 0.81 &
                                   tnrs.ncbi$Name_score > 0.51 & 
                                   tnrs.ncbi$Specific_epithet_score>=0.67)
length(index.species.2)

index.species.3 <- which(tnrs.ncbi$Name_matched_rank == "species" &
                                   tnrs.ncbi$Taxonomic_status == "No opinion"  &
                                   tnrs.ncbi$Genus_score > 0.7 &
                                   tnrs.ncbi$Specific_epithet_score > 0.75)
length(index.species.3)

index.species <- unique(c(index.species.1, index.species.2, index.species.3))
length(index.species)
```
### Variety level
```{r, eval = T}
index.var <- which((tnrs.ncbi$Name_matched_rank == "subspecies" |
                              tnrs.ncbi$Name_matched_rank == "unknown" |
                              tnrs.ncbi$Name_matched_rank == "variety") &
                             (tnrs.ncbi$Taxonomic_status == "Accepted" |
                              tnrs.ncbi$Taxonomic_status == "No opinion" |
                              tnrs.ncbi$Taxonomic_status == "Synonym"))
length(index.var)
```

### Select `certain` or `uncertain` names
```{r, eval = T}
index.ncbi <- unique(c(index.family, index.genus, index.species, index.var))

tnrs.ncbi.certain <- tnrs.ncbi[index.ncbi,]
nrow(tnrs.ncbi.certain)
write_csv(tnrs.ncbi.certain, path = "../_derived/TNRS_submit/tnrs.ncbi.certain.csv")

tnrs.ncbi.uncertain <- tnrs.ncbi[-index.ncbi,]
nrow(tnrs.ncbi.uncertain)
write_csv(tnrs.ncbi.uncertain, path = "../_derived/TNRS_submit/tnrs.ncbi.uncertain.csv")
save(tnrs.ncbi.certain, tnrs.ncbi.uncertain, file="../_derived/TNRS_submit/tnrs.iter3.RData")
```

After iteration 3, there are still `r nrow(tnrs.ncbi.uncertain)` unresolved taxa.




## Iteration 4 - Using `The Plant List` matching tools for unresolved names
Generate names list from `tnrs.ncbi.uncertain` to be matched against `The Plant List`, using `Taxonstand::TPL`.  

```{r, eval = F}
tpl.submit <- tnrs.ncbi.uncertain %>% dplyr::select(Name_submitted)
write_csv(tpl.submit, path="../_derived/TPL/tpl.submit.csv")

tpl.ncbi <- TPL(tpl.submit$Name_submitted)
write_csv(tpl.ncbi, path = "../_derived/TPL/tpl_results_iter4.csv")
```


```{r}
tpl.ncbi <- read_csv("../_derived/TPL/tpl_results_iter4.csv",  
         locale = locale(encoding = 'UTF-8'),quote="",
          col_type = cols(
            .default = col_character(),
              Hybrid.marker = col_logical(),
              Plant.Name.Index = col_logical(),
              TPL.version = col_double(),
              Typo = col_logical(),
              WFormat = col_logical(),
              Higher.level = col_logical(),
              Date = col_date(format = "")
            ))
tpl.ncbi.certain <- tpl.ncbi %>%
  filter(Plant.Name.Index==T)
nrow(tpl.ncbi.certain)
write_csv(tpl.ncbi.certain, path = "../_derived/TPL/tpl.ncbi.certain.csv")

tpl.ncbi.uncertain <- tpl.ncbi %>%
  filter(Plant.Name.Index==F) %>%
  dplyr::select(Taxon)
nrow(tpl.ncbi.uncertain)
write_csv(tpl.ncbi.uncertain, path = "../_derived/TPL/tpl.ncbi.uncertain.csv")

save(tpl.ncbi.certain, tpl.ncbi.uncertain, file="../_derived/TNRS_submit/tnrs.iter4.RData")
```


# Merge the resolved species lists into a Backbone
## Read files

```{r, eval = T}
load("../_derived/TNRS_submit/tnrs.iter1.RData")
load("../_derived/TNRS_submit/tnrs.iter2.RData")
load("../_derived/TNRS_submit/tnrs.iter3.RData")
load("../_derived/TNRS_submit/tnrs.iter4.RData")
```

Combine the `certain` data sets:
```{r, eval = T, warning=F}

Backbone <- spec.list.TRY.sPlot %>%
  as.tbl() %>%
  rename(Name_sPlot_TRY=OriginalNames, 
         Name_string_corr1=Species) %>%
  left_join(tnrs.submit.iter2 %>%
              dplyr::select(-new) %>%
              rename(Name_string_corr1=old, Name_string_corr2=Name_binomial),
            by="Name_string_corr1") %>%
  mutate(Name_submitted=ifelse(!is.na(Name_string_corr2), Name_string_corr2, Name_string_corr1)) %>%
  dplyr::select(Name_sPlot_TRY, Name_string_corr1, Name_string_corr2, Source, Name_submitted) %>%
  rename(sPlot_TRY=Source) %>%
  left_join(tnrs.res.certain %>% 
              bind_rows(tnrs.res.iter2.certain) %>%
              bind_rows(tnrs.ncbi.certain) %>%
  #reformat TPL output to tnrs output
              bind_rows(tpl.ncbi.certain %>%
                    rename(Name_submitted=Taxon,
                    Name_matched_url=ID,
                    Taxonomic_status=Taxonomic.status,
                    Accepted_name_author=New.Authority,
                    Accepted_name_url=New.ID,
                    Accepted_name_family=Family, 
                    Selected=Plant.Name.Index) %>%
                    mutate_at(.vars=vars(New.Hybrid.marker, New.Infraspecific.rank, New.Infraspecific),
                              .fun=~ifelse(is.na(.), "", .)) %>%
                    mutate(Accepted_name=paste(New.Genus, New.Hybrid.marker, 
                                               New.Species, New.Infraspecific.rank, 
                                               New.Infraspecific)) %>%
                    mutate(Accepted_name=gsub(pattern="\\s+", " ", Accepted_name)) %>%
                    mutate(Accepted_name_species=paste(New.Genus, New.Hybrid.marker, New.Species)) %>%
                    mutate(Accepted_name_species=gsub(pattern="\\s+", " ", Accepted_name_species)) %>%
                    mutate(Accepted_name_rank=ifelse(Higher.level==F, "species", NA)) %>%
                    mutate(Source=paste("tpl", TPL.version)) %>%
                    dplyr::select( (data.frame(colmatch=match(colnames(tnrs.ncbi), 
                                                        names(.))) %>%
                                filter(!is.na(colmatch)))$colmatch)) %>%
                    group_by(Name_submitted) %>%  #Some double matches. Prioritize first iterations
                    slice(1),
            by="Name_submitted")
#Double check
nrow(Backbone) == nrow(spec.list.TRY.sPlot)
```


## Tag unresolved names and create output columns
Add four additional columns.
If names were not corrected, set `Taxonomic.status == ""`, and assign `No suitable matches found.` to the remaining species.
```{r, eval = T}
Backbone <- Backbone %>%
  mutate(Status_correct=ifelse(Taxonomic_status %in% c("Accepted", "Synonym", "Unresolved"), 
                               Taxonomic_status, NA)) %>%
  mutate(Status_correct=replace(Status_correct, 
                                list=is.na(Status_correct), 
                                values="No suitable matches found.")) %>% 
  mutate(Status_correct=factor(Status_correct)) %>%
  #Create Name_correct field. Use Accepted names, if any. Otherwise matched names.
  mutate(Name_correct=ifelse(!is.na(Accepted_name), 
                             Accepted_name, 
                             Name_matched)) %>%
  mutate(Genus_correct=ifelse(!is.na(Name_correct) & (!Accepted_name_rank %in% c("family")), 
                      word(Name_correct,1), 
                      NA)) %>% 
  mutate(Name_correct=ifelse(!is.na(Name_correct), 
                             Name_correct, 
                             "No suitable matches found.")) %>%
  mutate(Rank_correct=ifelse(!is.na(Name_matched_rank),
                             as.character(Name_matched_rank), 
                             "higher")) %>%
  mutate(Rank_correct=factor(Rank_correct, levels=c("higher", "family", "genus", "species",
                                                   "subspecies", "variety", "infraspecies",
                                                   "race", "forma")
                             )) %>% 
  mutate(Name_short=ifelse(!is.na(Accepted_name_species), Accepted_name_species, NA))

summary(Backbone$Status_correct)
summary(Backbone$Rank_correct)
```



## Complete list of families
### Derive info from other species of the same Genera in the Backbone itself
Copy family info for taxa resolved at family level
```{r}
Backbone <- Backbone %>% 
  mutate(family.lev=str_extract(word(Name_correct,1), pattern='([^\\s]+aceae)')) %>%
  mutate(Family_correct=ifelse(!is.na(Accepted_name_family), 
                                     Accepted_name_family, 
                                     family.lev)) %>% 
  dplyr::select(-family.lev) 

#Records with missing family info
sum((is.na(Backbone$Family_correct)))
```

Derive family info from each genus in the backbone, and use this info to complement records from the same genera, but with missing family info.
```{r}
genera_families <- Backbone %>% 
  filter(Taxonomic_status=="Accepted") %>% 
  dplyr::select(Genus_correct, Family_correct) %>% 
  rename(family=Family_correct) %>% 
  distinct() %>% 
  na.omit() %>% 
  #for some genera there are multiple families assigned 
  # (e.g. in case of unresolved species names )
  # Extract the family names that occurs the most across each genus
  group_by(Genus_correct, family) %>% 
  summarize(n=n()) %>% 
  arrange(desc(n)) %>% 
  slice(1) %>% 
  ungroup() %>% 
  dplyr::select(-n)

# Assign family derived from backbone to other records
Backbone <- Backbone %>% 
  left_join(genera_families, by="Genus_correct") %>% 
  mutate(Family_correct=ifelse( (is.na(Family_correct) & !is.na(family)),
                                       family, 
                                       Family_correct)) %>% 
  dplyr::select(-family)

#Records with missing family info
sum(is.na(Backbone$Family_correct))
```


### Resolve genera with missing family info with `TNRS`
```{r, eval=F}

Genera_submit <- Backbone %>% 
  filter(is.na(Family_correct))  %>% 
  dplyr::select(Genus_correct) %>% 
  distinct()

write_csv(Genera_submit, "../_derived/TNRS_submit/Genera_submit.csv")
```

Import results from TNRS. Best match only and simple download
```{r}
import.profile <- cols(
  Name_submitted = col_character(),
  Name_matched = col_character(),
  Author_matched = col_logical(),
  Overall_score = col_double(),
  Taxonomic_status = col_character(),
  Accepted_name = col_character(),
  Accepted_author = col_character(),
  Accepted_family = col_character(),
  Source = col_character(),
  Warnings = col_character(),
  Accepted_name_lsid = col_character()
)
      
tnrs.genera <- read_delim("../_derived/TNRS_submit/tnrs_genera.txt", delim="\t",  
         locale = locale(encoding = 'UTF-8'),quote="",col_type = import.profile)
```
Attach resolved families to backbone
```{r}
Backbone <- Backbone %>% 
  left_join(tnrs.genera %>%
              dplyr::select(Name_submitted, Accepted_family) %>% 
              rename(Genus_correct=Name_submitted, Family_import=Accepted_family), 
            by="Genus_correct") %>% 
  mutate(Family_correct=ifelse(is.na(Family_correct), 
                                Family_import, 
                                Family_correct)) %>%
  dplyr::select(-Family_import)
  
#Records with missing family info
sum(is.na(Backbone$Family_correct))
```

### Complement with data from `The Catalogue of Life`.
```{r, eval=F}
#Download data from Catalogue of Life - 2019
download.file("http://www.catalogueoflife.org/DCA_Export/zip/archive-kingdom-plantae-bl3.zip",
              destfile="/data/sPlot/users/Francesco/Ancillary_Data/Catalogue_of_Life/CatLife2019.zip")
unzip("/data/sPlot/users/Francesco/Ancillary_Data/Catalogue_of_Life/CatLife2019.zip", files="taxa.txt", exdir = "/data/sPlot/users/Francesco/Ancillary_Data/Catalogue_of_Life/")

```



```{r, message=F, warning=F}
cat.life <- read_delim("/data/sPlot/users/Francesco/Ancillary_Data/Catalogue_of_Life/taxa.txt", 
                       delim="\t")
Genera_missing <- Backbone %>%
  filter(is.na(Family_correct) & !is.na(Genus_correct)) %>%
  dplyr::select(Genus_correct) %>% 
  distinct()

Backbone <- Backbone %>% 
  left_join(cat.life %>%
      dplyr::select(genus, family) %>% 
      distinct() %>% 
      filter(family != "") %>% 
      filter(genus %in% Genera_missing$Genus_correct) %>% 
      rename(Genus_correct=genus), 
    by="Genus_correct") %>% 
  mutate(Family_correct=ifelse(is.na(Family_correct) & !is.na(family), 
                                family, 
                                Family_correct)) %>%
  dplyr::select(-family)

#Records with missing family info
sum(is.na(Backbone$Family_correct))
```
After matching the remaining genera with the Catalogue of life there are still `r nrow(Backbone %>% filter(is.na(Family_correct)))` records without Family affiliation, for a total of `r nrow(Backbone %>% filter(is.na(Family_correct)) %>% dplyr::select(Genus_correct) %>% distinct())` genera.  
  
Manually fix some residual, known issues:
```{r}
Backbone <- Backbone %>%
  mutate(Family_correct=replace(Family_correct, 
                                list=word(Name_short, 1)=="Coptidium",
                                values="Ranunculaceae"))

```



## Create Field `is_vascular_plant`
Assign all families that belong to `Tracheophyta` to category `is_vascular_species`, based on The Catalogue of Life
```{r}
Backbone <- Backbone %>% 
  left_join(cat.life %>% 
              dplyr::select(phylum, family) %>% 
              distinct() %>% 
              na.omit() %>% 
              rename(Family_correct=family), 
            by="Family_correct") %>% 
  mutate(is_vascular_species=ifelse(phylum=="Tracheophyta", T, F))

table(Backbone$is_vascular_species, exclude=NULL)
```
## Export Backbone
```{r}
save(Backbone, file="../_output/Backbone3.0.RData")
```

## Export species list to request in TRY
```{r}
ToSubmit1 <- Backbone %>% 
  filter(grepl(sPlot_TRY, pattern = "S")) %>% 
  filter(grepl(sPlot_TRY, pattern = "T")) %>% 
  dplyr::select(Name_sPlot_TRY) %>% 
  rename(Name_submit = Name_sPlot_TRY)

## add names from T, which match names from S only after standardization
ToSubmit2 <- Backbone %>% 
  filter(grepl(sPlot_TRY, pattern = "T")) %>% 
  filter(!grepl(sPlot_TRY, pattern = "S")) %>% 
  filter(Name_sPlot_TRY  %in% ((Backbone %>% 
                                  filter(grepl(sPlot_TRY, pattern = "S")) %>% 
                                  filter(!grepl(sPlot_TRY, pattern = "T")) %>% 
                                  dplyr::select(Name_correct) %>% 
                                  distinct())$Name_correct)) %>% 
  dplyr::select(Name_sPlot_TRY) %>% 
  rename(Name_submit = Name_sPlot_TRY)

ToSubmit <- ToSubmit1 %>% 
  bind_rows(ToSubmit2)

write_csv(ToSubmit, "../_output/Submit_TRY.csv")
```



# Statistics
## Statistics for backbone combining names in `sPlot3.0` and `TRY5.0`

### All taxon name entries

```{r, eval = T}
load("../_output/Backbone3.0.RData")
```
**How many new entries are in the backbone 3.0 compared to the backbone 2.1? How many entries are in common?**

```{r, eval = T, echo=F}
load("/data/sPlot/releases/sPlot2.1/backbone.splot2.1.try3.is.vascular.Rdata")

incommon <- nrow(Backbone %>% 
  dplyr::select(Name_sPlot_TRY) %>% 
  inner_join(backbone.splot2.1.try3 %>% 
               dplyr::select(names.sPlot.TRY) %>% 
               rename(Name_sPlot_TRY=names.sPlot.TRY),
                 by="Name_sPlot_TRY"))
```
The new backbone contains `r nrow(Backbone)`. The backbone 2.1 contained `r nrow(backbone.splot2.1.try3)`. The two backbones have `r incommon` records in common.

**Database affiliations (`sPlot 3.1`,  `TRY 3.0`, and `Alpine`).**

```{r, eval = T, echo=F}
kable((table(Backbone$sPlot_TRY)), caption = "Number of (standardized) name entries
unique to, or shared between sPlot (S), TRY (T) and Alpine (A).") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

`r nrow(Backbone %>% filter(sPlot_TRY %in% c("S", "ST", "SA", "STA")))` of the total number of entries belong to sPlot. `r nrow(Backbone %>% filter(sPlot_TRY %in% c("T", "ST", "TA", "STA")))` name entries belong to TRY.

**Taxonomic ranks:**
```{r, eval = T, echo=F}
kable((table(Backbone$Rank_correct, exclude=NULL)), caption = "Number of (standardized) name entries per taxonomic rank.") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

**Taxonomic status:**
```{r, eval = T, echo=F}
kable((table(Backbone$Taxonomic_status, exclude=NULL)), caption = "Number of (standardized) name entries for taxonomic status")%>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

**Total number of unique standardized taxon names and families:**
```{r, eval = T}
length(unique(Backbone$Name_short))-1 # minus 1 for NA
length(unique(Backbone$Family_correct))-1 # minus 1 for NA
```

**Number of entries corresponding to vascular plant species:**
```{r, eval = T}
table(Backbone$is_vascular_species, exclude=NULL)
```

**Number of duplicated entries after taxonomic standardization:**
Frequency of original (non-standardized) species names per resolved (non-standardized) name (excluding non-vascular and non-matched species).
```{r, eval = T}
df.count <- Backbone %>%
    dplyr::filter(is_vascular_species == TRUE & !is.na(Name_correct)) %>%
    dplyr::group_by(Name_correct) %>%
    dplyr::summarise(n = n()) %>%
    dplyr::arrange(desc(n))
```

```{r, echo=F}

kable(df.count[c(1:20), ], , caption = "Number of unresolved, original name
entries per resolved name. (Only first 20 shown") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")

```




### Based on `unique` standardized names
Generate version of the backbone that only includes the unique resolved names in `Name.short`, and for the non-unique names, the first rows of duplicated name:

```{r, eval = T}
Backbone.uni <- Backbone %>% 
  distinct(Name_short, .keep_all = T) %>%
  filter(!is.na(Name_short))
```
There are `r nrow(Backbone.uni)` unique taxon names the in the backbone.   

Exclude the non-vascular plant and non-matching taxon names:

```{r, eval = T}
Backbone.uni.vasc <- Backbone.uni %>%
    dplyr::filter(is_vascular_species == TRUE)

```
`df.uni` had two names less than `df.count`, as they were accidentially tagged as non-vascular species names. Resolve that issue:

**Now, run the stats for unique resolved names (excluding non-vascular and non-matching taxa):**
```{r, eval = T}
length(Backbone.uni.vasc$Name_short)
```

There are `r length(Backbone.uni.vasc$Name.short)` unique (vascular plant) taxon names:

```{r, eval = T, echo=F}
kable((table(Backbone.uni.vasc$sPlot_TRY)), caption = "Number of (standardized) vascular plant taxon names per unique to, and shared between TRY (S), sPlot (T) and the Alpine (A) dataset.") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

**Taxonomic ranks:**
```{r, eval = T, echo=F}
kable((table(Backbone.uni.vasc$Rank_correct, exclude=NULL)), caption = "Number of (standardized) name entries per taxonomic rank.") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")

```

**Taxonomic status:**
```{r, eval = T, echo=F}
kable((table(Backbone.uni.vasc$Status_correct, exclude=NULL)), caption = "Number of (standardized) name entries per taxonomic status")%>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

**Total number of unique standardized taxon names and families:**
```{r, eval = T}
length(unique(Backbone.uni.vasc$Name_short))-1 # minus 1 for NA
length(unique(Backbone.uni.vasc$Family_correct))-1
```

## Stats for the corrected names in `sPlot` only:
```{r, eval = T}
Backbone.uni.sPlot <- Backbone.uni.vasc %>%
  filter(sPlot_TRY %in% c("S", "ST", "SA", "STA"))
```

There are `r nrow(Backbone.uni.sPlot %>% distinct(Name_correct))` unique, corrected names of vascular plants for sPlot species

**Database affiliations**
```{r, eval = T, echo=F}
kable((table(Backbone.uni.sPlot$sPlot_TRY)), caption = "Number of (standardized) vascular
plant taxon names per unique to sPlot (S), and shared with TRY (ST), the Alpine dataset (SA) or both (STA).")%>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")
```

**Taxonomic ranks:**
```{r, eval = T, echo=F}
kable((table(Backbone.uni.sPlot$Rank_correct, exclude=NULL)), caption = "Number of (standardized) vascular plant taxon names per taxonomic rank.")%>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")

```

**Taxonomic status:**
```{r, eval = T, echo=F}
kable((table(Backbone.uni.sPlot$Status_correct, exclude=NULL)), caption = "Number of (standardized) vascular plant taxon names that correspond to `Accepted`, `Synonyms` or Unresolved species, respecively.") %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive"), 
                  latex_options = "basic", 
                  full_width = F, position = "center")

```

**Number of families in sPlot**:
```{r, eval = T}
length(unique(Backbone$Family_correct))
```

**Done!**

--------------------

# `R`-settings

```{r, eval = T}
sessionInfo()
```

```{r, eval=F, echo=F}
toCheck_manual <- read_csv("../_derived/TPL/tpl.ncbi.uncertain.csv")
load("../_output/Backbone3.0.RData")

toCheck_manual <- toCheck_manual %>% 
  rename(Name_string_corr2=Taxon) %>% 
  left_join(Backbone %>% 
              dplyr::select(Name_sPlot_TRY:sPlot_TRY), 
            by="Name_string_corr2") %>% 
  left_join(DT0 %>% 
              rename(Name_sPlot_TRY=`Matched concept`) %>% 
              dplyr::select(Name_sPlot_TRY) %>% 
              group_by(Name_sPlot_TRY) %>% 
              summarize(count=n()),
            by="Name_sPlot_TRY") %>% 
  left_join(DT0 %>% 
              rename(Name_sPlot_TRY=`Matched concept`) %>% 
              dplyr::select(Name_sPlot_TRY, Taxonomy) %>%
              distinct(Name_sPlot_TRY, .keep_all=T), 
            by="Name_sPlot_TRY"
              )

write_csv(toCheck_manual, path="../_derived/TPL/toCheck_Manual.csv")

```


