% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/replace_tokens.R
\name{replace_tokens}
\alias{replace_tokens}
\title{Replace Tokens}
\usage{
replace_tokens(x, tokens, replacement = NULL, ignore.case = FALSE, ...)
}
\arguments{
\item{x}{A character vector.}

\item{tokens}{A vector of token to be replaced.}

\item{replacement}{A single character string to replace the tokens with.
The default, \code{NULL}, replaces the tokens with nothing.}

\item{ignore.case}{logical.  If \code{TRUE} the case of the tokens will 
be ignored.}

\item{\ldots}{ignored.}
}
\value{
Returns a vector of strings with tokens replaced.
}
\description{
Replace tokens with a single substring.  This is much faster than 
\code{\link[textclean]{mgsub}} if one wants to replace fixed tokens
with a single value or remove them all together.  This can be useful
for quickly replacing tokens like names in string with a single
value in order to reduce noise.
}
\note{
The function splits the string apart into tokens for speed
optimization.  After the replacement occurs the strings are pasted back
together.  The strings are not guaranteed to retain exact spacing of the
original.
}
\examples{
replace_tokens(DATA$state, c('No', 'what', "it's"))
replace_tokens(DATA$state, c('No', 'what', "it's"), "<<TOKEN>>")
replace_tokens(
    DATA$state, 
    c('No', 'what', "it's"), 
    "<<TOKEN>>", 
    ignore.case = TRUE
)

\dontrun{
## Now let's see the speed
## Set up data
library(textshape)
data(hamlet)
set.seed(11)
tokens <- sample(unique(unlist(split_token(hamlet$dialogue))), 2000)

tic <- Sys.time()
head(replace_tokens(hamlet$dialogue, tokens))
(toc <- Sys.time() - tic)


tic <- Sys.time()
head(mgsub(hamlet$dialogue, tokens, ""))
(toc <- Sys.time() - tic)


## Amp it up 20x more data
tic <- Sys.time()
head(replace_tokens(rep(hamlet$dialogue, 20), tokens))
(toc <- Sys.time() - tic)

## Replace names example

library(lexicon)
library(textshape)
nms <- gsub("(^.)(.*)", "\\\\U\\\\1\\\\L\\\\2", common_names, perl = TRUE)
x <- split_portion(
    sample(c(sample(grady_augmented, 5000), sample(nms, 10000, TRUE))), 
    n.words = 12
)
x$text.var <- paste0(
    x$text.var, 
    sample(c('.', '!', '?'), length(x$text.var), TRUE)
 )
replace_tokens(x$text.var, nms, 'NAME')
}
}
\seealso{
\code{\link[textclean]{mgsub}}, \code{\link[textclean]{match_tokens}}
}
