diff options
Diffstat (limited to 'unicode-segmentation/src/lib.rs')
| -rw-r--r-- | unicode-segmentation/src/lib.rs | 242 |
1 files changed, 0 insertions, 242 deletions
diff --git a/unicode-segmentation/src/lib.rs b/unicode-segmentation/src/lib.rs deleted file mode 100644 index fce3c52..0000000 --- a/unicode-segmentation/src/lib.rs +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Iterators which split strings on Grapheme Cluster, Word or Sentence boundaries, according -//! to the [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) rules. -//! -//! ```rust -//! extern crate unicode_segmentation; -//! -//! use unicode_segmentation::UnicodeSegmentation; -//! -//! fn main() { -//! let s = "a̐éö̲\r\n"; -//! let g = UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>(); -//! let b: &[_] = &["a̐", "é", "ö̲", "\r\n"]; -//! assert_eq!(g, b); -//! -//! let s = "The quick (\"brown\") fox can't jump 32.3 feet, right?"; -//! let w = s.unicode_words().collect::<Vec<&str>>(); -//! let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"]; -//! assert_eq!(w, b); -//! -//! let s = "The quick (\"brown\") fox"; -//! let w = s.split_word_bounds().collect::<Vec<&str>>(); -//! let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", " ", "fox"]; -//! assert_eq!(w, b); -//! } -//! ``` -//! -//! # no_std -//! -//! unicode-segmentation does not depend on libstd, so it can be used in crates -//! with the `#![no_std]` attribute. -//! -//! # crates.io -//! -//! You can use this package in your project by adding the following -//! to your `Cargo.toml`: -//! -//! ```toml -//! [dependencies] -//! unicode-segmentation = "1.3.0" -//! ``` - -#![deny(missing_docs, unsafe_code)] -#![doc(html_logo_url = "https://unicode-rs.github.io/unicode-rs_sm.png", - html_favicon_url = "https://unicode-rs.github.io/unicode-rs_sm.png")] - -#![no_std] - -#[cfg(test)] -#[macro_use] -extern crate std; - -#[cfg(test)] -#[macro_use] -extern crate quickcheck; - -pub use grapheme::{Graphemes, GraphemeIndices}; -pub use grapheme::{GraphemeCursor, GraphemeIncomplete}; -pub use tables::UNICODE_VERSION; -pub use word::{UWordBounds, UWordBoundIndices, UnicodeWords}; -pub use sentence::{USentenceBounds, USentenceBoundIndices, UnicodeSentences}; - -mod grapheme; -mod tables; -mod word; -mod sentence; - -#[cfg(test)] -mod test; -#[cfg(test)] -mod testdata; - -/// Methods for segmenting strings according to -/// [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/). -pub trait UnicodeSegmentation { - /// Returns an iterator over the [grapheme clusters][graphemes] of `self`. - /// - /// [graphemes]: http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries - /// - /// If `is_extended` is true, the iterator is over the - /// *extended grapheme clusters*; - /// otherwise, the iterator is over the *legacy grapheme clusters*. - /// [UAX#29](http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) - /// recommends extended grapheme cluster boundaries for general processing. - /// - /// # Examples - /// - /// ``` - /// # use self::unicode_segmentation::UnicodeSegmentation; - /// let gr1 = UnicodeSegmentation::graphemes("a\u{310}e\u{301}o\u{308}\u{332}", true) - /// .collect::<Vec<&str>>(); - /// let b: &[_] = &["a\u{310}", "e\u{301}", "o\u{308}\u{332}"]; - /// - /// assert_eq!(&gr1[..], b); - /// - /// let gr2 = UnicodeSegmentation::graphemes("a\r\nb🇷🇺🇸🇹", true).collect::<Vec<&str>>(); - /// let b: &[_] = &["a", "\r\n", "b", "🇷🇺", "🇸🇹"]; - /// - /// assert_eq!(&gr2[..], b); - /// ``` - fn graphemes<'a>(&'a self, is_extended: bool) -> Graphemes<'a>; - - /// Returns an iterator over the grapheme clusters of `self` and their - /// byte offsets. See `graphemes()` for more information. - /// - /// # Examples - /// - /// ``` - /// # use self::unicode_segmentation::UnicodeSegmentation; - /// let gr_inds = UnicodeSegmentation::grapheme_indices("a̐éö̲\r\n", true) - /// .collect::<Vec<(usize, &str)>>(); - /// let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")]; - /// - /// assert_eq!(&gr_inds[..], b); - /// ``` - fn grapheme_indices<'a>(&'a self, is_extended: bool) -> GraphemeIndices<'a>; - - /// Returns an iterator over the words of `self`, separated on - /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries). - /// - /// Here, "words" are just those substrings which, after splitting on - /// UAX#29 word boundaries, contain any alphanumeric characters. That is, the - /// substring must contain at least one character with the - /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic) - /// property, or with - /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values). - /// - /// # Example - /// - /// ``` - /// # use self::unicode_segmentation::UnicodeSegmentation; - /// let uws = "The quick (\"brown\") fox can't jump 32.3 feet, right?"; - /// let uw1 = uws.unicode_words().collect::<Vec<&str>>(); - /// let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"]; - /// - /// assert_eq!(&uw1[..], b); - /// ``` - fn unicode_words<'a>(&'a self) -> UnicodeWords<'a>; - - /// Returns an iterator over substrings of `self` separated on - /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries). - /// - /// The concatenation of the substrings returned by this function is just the original string. - /// - /// # Example - /// - /// ``` - /// # use self::unicode_segmentation::UnicodeSegmentation; - /// let swu1 = "The quick (\"brown\") fox".split_word_bounds().collect::<Vec<&str>>(); - /// let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", " ", "fox"]; - /// - /// assert_eq!(&swu1[..], b); - /// ``` - fn split_word_bounds<'a>(&'a self) -> UWordBounds<'a>; - - /// Returns an iterator over substrings of `self`, split on UAX#29 word boundaries, - /// and their offsets. See `split_word_bounds()` for more information. - /// - /// # Example - /// - /// ``` - /// # use self::unicode_segmentation::UnicodeSegmentation; - /// let swi1 = "Brr, it's 29.3°F!".split_word_bound_indices().collect::<Vec<(usize, &str)>>(); - /// let b: &[_] = &[(0, "Brr"), (3, ","), (4, " "), (5, "it's"), (9, " "), (10, "29.3"), - /// (14, "°"), (16, "F"), (17, "!")]; - /// - /// assert_eq!(&swi1[..], b); - /// ``` - fn split_word_bound_indices<'a>(&'a self) -> UWordBoundIndices<'a>; - - /// Returns an iterator over substrings of `self` separated on - /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries). - /// - /// The concatenation of the substrings returned by this function is just the original string. - fn unicode_sentences<'a>(&'a self) -> UnicodeSentences<'a>; - - /// Returns an iterator over substrings of `self` separated on - /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries). - /// - /// Here, "sentences" are just those substrings which, after splitting on - /// UAX#29 sentence boundaries, contain any alphanumeric characters. That is, the - /// substring must contain at least one character with the - /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic) - /// property, or with - /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values). - fn split_sentence_bounds<'a>(&'a self) -> USentenceBounds<'a>; - - /// Returns an iterator over substrings of `self`, split on UAX#29 sentence boundaries, - /// and their offsets. See `split_sentence_bounds()` for more information. - fn split_sentence_bound_indices<'a>(&'a self) -> USentenceBoundIndices<'a>; -} - -impl UnicodeSegmentation for str { - #[inline] - fn graphemes(&self, is_extended: bool) -> Graphemes { - grapheme::new_graphemes(self, is_extended) - } - - #[inline] - fn grapheme_indices(&self, is_extended: bool) -> GraphemeIndices { - grapheme::new_grapheme_indices(self, is_extended) - } - - #[inline] - fn unicode_words(&self) -> UnicodeWords { - word::new_unicode_words(self) - } - - #[inline] - fn split_word_bounds(&self) -> UWordBounds { - word::new_word_bounds(self) - } - - #[inline] - fn split_word_bound_indices(&self) -> UWordBoundIndices { - word::new_word_bound_indices(self) - } - - #[inline] - fn unicode_sentences(&self) -> UnicodeSentences { - sentence::new_unicode_sentences(self) - } - - #[inline] - fn split_sentence_bounds(&self) -> USentenceBounds { - sentence::new_sentence_bounds(self) - } - - #[inline] - fn split_sentence_bound_indices(&self) -> USentenceBoundIndices { - sentence::new_sentence_bound_indices(self) - } -} |