WIP: handling extremely long lines with good performance.

The formatters now work on grapheme iterators instead of directly on
lines, which frees up the LineFormatter to break up long lines into
smaller blocks of text.  This is partially taken advantage of right
now in various parts of the code, but more work is still needed to
get it both working properly and fast.

src/formatter.rs

@@ -1,8 +1,14 @@
 #![allow(dead_code)]
 
-use buffer::line::Line;
 use buffer::Buffer;
 
+// Maximum graphemes in a line before a soft line break is forced.
+// This is necessary to prevent pathological formatting cases which
+// could slow down the editor arbitrarily for arbitrarily long
+// lines.
+pub const LINE_BLOCK_LENGTH: usize = 4096;
+
+
 #[derive(Copy, PartialEq)]
 pub enum RoundingBehavior {
     Round,
@@ -14,23 +20,40 @@ pub enum RoundingBehavior {
 pub trait LineFormatter {
     fn single_line_height(&self) -> usize;
     
-    /// Returns the 2d visual dimensions of the given line when formatted
+    /// Returns the 2d visual dimensions of the given text when formatted
     /// by the formatter.
-    fn dimensions(&self, line: &Line) -> (usize, usize);
+    /// The text to be formatted is passed as a grapheme iterator.
+    fn dimensions<'a, T>(&'a self, g_iter: T) -> (usize, usize)
+    where T: Iterator<Item=&'a str>;
     
     
-    /// Converts a grapheme index within a line into a visual 2d position.
+    /// Converts a grapheme index within a text into a visual 2d position.
-    fn index_to_v2d(&self, line: &Line, index: usize) -> (usize, usize);
+    /// The text to be formatted is passed as a grapheme iterator.
+    fn index_to_v2d<'a, T>(&'a self, g_iter: T, index: usize) -> (usize, usize)
+    where T: Iterator<Item=&'a str>;
     
     
-    /// Converts a visual 2d position into a grapheme index within a line.
+    /// Converts a visual 2d position into a grapheme index within a text.
-    fn v2d_to_index(&self, line: &Line, v2d: (usize, usize), rounding: (RoundingBehavior, RoundingBehavior)) -> usize;
+    /// The text to be formatted is passed as a grapheme iterator.
+    fn v2d_to_index<'a, T>(&'a self, g_iter: T, v2d: (usize, usize), rounding: (RoundingBehavior, RoundingBehavior)) -> usize
+    where T: Iterator<Item=&'a str>;
 
 
     fn index_to_horizontal_v2d(&self, buf: &Buffer, index: usize) -> usize {
         let (line_i, col_i) = buf.index_to_line_col(index);
         let line = buf.get_line(line_i);
-        return self.index_to_v2d(line, col_i).1;
+        
+        // Find the right block in the line, and the index within that block
+        let mut index: usize = col_i;
+        let mut line_block: usize = 0;
+        while col_i >= LINE_BLOCK_LENGTH {
+            line_block += 1;
+            index -= LINE_BLOCK_LENGTH; 
+        }
+        
+        // Get an iter into the right block
+        let g_iter = line.grapheme_iter_at_index(line_block * LINE_BLOCK_LENGTH);
+        return self.index_to_v2d(g_iter, index).1;
     }
     
     
@@ -40,14 +63,14 @@ pub trait LineFormatter {
         // TODO: handle rounding modes
         // TODO: do this with bidirectional line iterator
         let (mut line_i, mut col_i) = buf.index_to_line_col(index);
-        let (mut y, x) = self.index_to_v2d(buf.get_line(line_i), col_i);
+        let (mut y, x) = self.index_to_v2d(buf.get_line(line_i).grapheme_iter(), col_i);
         let mut new_y = y as isize + offset;
         
         // First, find the right line while keeping track of the vertical offset
         let mut line;
         loop {
             line = buf.get_line(line_i);
-            let (h, _) = self.dimensions(line);
+            let (h, _) = self.dimensions(line.grapheme_iter());
             
             if new_y >= 0 && new_y < h as isize {
                 y = new_y as usize;
@@ -71,7 +94,7 @@ pub trait LineFormatter {
                     
                     line_i -= 1;
                     line = buf.get_line(line_i);
-                    let (h, _) = self.dimensions(line);
+                    let (h, _) = self.dimensions(line.grapheme_iter());
                     new_y += h as isize;
                 }
                 else {
@@ -82,7 +105,7 @@ pub trait LineFormatter {
         
         // Next, convert the resulting coordinates back into buffer-wide
         // coordinates.
-        col_i = self.v2d_to_index(line, (y, x), rounding);
+        col_i = self.v2d_to_index(line.grapheme_iter(), (y, x), rounding);
         
         return buf.line_col_to_index((line_i, col_i));
     }
@@ -92,8 +115,18 @@ pub trait LineFormatter {
         let (line_i, col_i) = buf.index_to_line_col(index);
         let line = buf.get_line(line_i);
         
-        let (v, _) = self.index_to_v2d(line, col_i);
+        // Find the right block in the line, and the index within that block
-        let mut new_col_i = self.v2d_to_index(line, (v, horizontal), (RoundingBehavior::Floor, rounding));
+        let mut col_i_adjusted: usize = col_i;
+        let mut line_block: usize = 0;
+        while col_i >= LINE_BLOCK_LENGTH {
+            line_block += 1;
+            col_i_adjusted -= LINE_BLOCK_LENGTH; 
+        }
+        let start_index = line_block * LINE_BLOCK_LENGTH;
+        
+        // Calculate the horizontal position
+        let (v, _) = self.index_to_v2d(line.grapheme_iter_at_index(start_index), col_i_adjusted);
+        let mut new_col_i = start_index + self.v2d_to_index(line.grapheme_iter_at_index(start_index), (v, horizontal), (RoundingBehavior::Floor, rounding));
         
         // Make sure we're not pushing the index off the end of the line
         if (line_i + 1) < buf.line_count()

src/term_ui/formatter.rs

@@ -1,7 +1,6 @@
 use std::cmp::max;
 
 use string_utils::{is_line_ending};
-use buffer::line::{Line, LineGraphemeIter};
 use formatter::{LineFormatter, RoundingBehavior};
 
 //===================================================================
@@ -22,9 +21,11 @@ impl ConsoleLineFormatter {
         }
     }
     
-    pub fn iter<'a>(&'a self, line: &'a Line) -> ConsoleLineFormatterVisIter<'a> {
+    pub fn iter<'a, T>(&'a self, g_iter: T) -> ConsoleLineFormatterVisIter<'a, T>
-        ConsoleLineFormatterVisIter {
+    where T: Iterator<Item=&'a str>
-            grapheme_iter: line.grapheme_iter(),
+    {
+        ConsoleLineFormatterVisIter::<'a, T> {
+            grapheme_iter: g_iter,
             f: self,
             pos: (0, 0),
         }
@@ -38,10 +39,12 @@ impl LineFormatter for ConsoleLineFormatter {
     }
     
     
-    fn dimensions(&self, line: &Line) -> (usize, usize) {
+    fn dimensions<'a, T>(&'a self, g_iter: T) -> (usize, usize)
+    where T: Iterator<Item=&'a str>
+    {
         let mut dim: (usize, usize) = (0, 0);
         
-        for (_, pos, width) in self.iter(line) {       
+        for (_, pos, width) in self.iter(g_iter) {       
             dim = (max(dim.0, pos.0), max(dim.1, pos.1 + width));
         }
         
@@ -51,12 +54,14 @@ impl LineFormatter for ConsoleLineFormatter {
     }
     
     
-    fn index_to_v2d(&self, line: &Line, index: usize) -> (usize, usize) {
+    fn index_to_v2d<'a, T>(&'a self, g_iter: T, index: usize) -> (usize, usize)
+    where T: Iterator<Item=&'a str>
+    {
         let mut pos = (0, 0);
         let mut i = 0;
         let mut last_width = 0;
         
-        for (_, _pos, width) in self.iter(line) {
+        for (_, _pos, width) in self.iter(g_iter) {
             pos = _pos;
             last_width = width;
             i += 1;
@@ -70,11 +75,13 @@ impl LineFormatter for ConsoleLineFormatter {
     }
     
     
-    fn v2d_to_index(&self, line: &Line, v2d: (usize, usize), _: (RoundingBehavior, RoundingBehavior)) -> usize {
+    fn v2d_to_index<'a, T>(&'a self, g_iter: T, v2d: (usize, usize), _: (RoundingBehavior, RoundingBehavior)) -> usize
+    where T: Iterator<Item=&'a str>
+    {
         // TODO: handle rounding modes
         let mut i = 0;
         
-        for (_, pos, _) in self.iter(line) {
+        for (_, pos, _) in self.iter(g_iter) {
             if pos.0 > v2d.0 {
                 break;
             }
@@ -94,15 +101,19 @@ impl LineFormatter for ConsoleLineFormatter {
 // An iterator that iterates over the graphemes in a line in a
 // manner consistent with the ConsoleFormatter.
 //===================================================================
-pub struct ConsoleLineFormatterVisIter<'a> {
+pub struct ConsoleLineFormatterVisIter<'a, T>
-    grapheme_iter: LineGraphemeIter<'a>,
+where T: Iterator<Item=&'a str>
+{
+    grapheme_iter: T,
     f: &'a ConsoleLineFormatter,
     pos: (usize, usize),
 }
 
 
 
-impl<'a> Iterator for ConsoleLineFormatterVisIter<'a> {
+impl<'a, T> Iterator for ConsoleLineFormatterVisIter<'a, T>
+where T: Iterator<Item=&'a str>
+{
     type Item = (&'a str, (usize, usize), usize);
 
     fn next(&mut self) -> Option<(&'a str, (usize, usize), usize)> {

src/term_ui/mod.rs

@@ -3,7 +3,7 @@
 use rustbox;
 use rustbox::Color;
 use editor::Editor;
-use formatter::LineFormatter;
+use formatter::{LineFormatter, LINE_BLOCK_LENGTH};
 use std::char;
 use std::time::duration::Duration;
 use string_utils::{is_line_ending};
@@ -103,7 +103,7 @@ impl TermUI {
             let mut e = self.rb.poll_event(); // Block until we get an event
             loop {
                 match e {
-                    Ok(rustbox::Event::KeyEvent(modifier, key, character)) => {
+                    Ok(rustbox::Event::KeyEvent(_, key, character)) => {
                         //println!("      {} {} {}", modifier, key, character);
                         match key {
                             K_CTRL_Q => {
@@ -359,7 +359,7 @@ impl TermUI {
         let gutter_width = editor.editor_dim.1 - editor.view_dim.1;
         let (starting_line, _) = editor.buffer.index_to_line_col(editor.view_pos.0);
         let mut grapheme_index = editor.buffer.line_col_to_index((starting_line, 0));
-        let (vis_line_offset, _) = editor.formatter.index_to_v2d(editor.buffer.get_line(starting_line), editor.view_pos.0 - grapheme_index);
+        let (vis_line_offset, _) = editor.formatter.index_to_v2d(editor.buffer.get_line(starting_line).grapheme_iter(), editor.view_pos.0 - grapheme_index);
         
         let mut screen_line = c1.0 as isize - vis_line_offset as isize;
         let screen_col = c1.1 as isize + gutter_width as isize;
@@ -382,60 +382,82 @@ impl TermUI {
             
             // Loop through the graphemes of the line and print them to
             // the screen.
+            let mut line_g_index: usize = 0;
+            let mut line_block_index: usize = 0;
             let mut last_pos_y = 0;
-            for (g, (pos_y, pos_x), width) in editor.formatter.iter(line) {
+            let mut lines_traversed: usize = 0;
-                last_pos_y = pos_y;
+            let mut g_iter = editor.formatter.iter(line.grapheme_iter());
-                // Calculate the cell coordinates at which to draw the grapheme
+            loop {
-                let px = pos_x as isize + screen_col - editor.view_pos.1 as isize;
+                if let Some((g, (pos_y, pos_x), width)) = g_iter.next() {
-                let py = pos_y as isize + screen_line;
+                    if last_pos_y != pos_y {
-                
+                        if last_pos_y < pos_y {
-                // If we're off the bottom, we're done
+                            lines_traversed += pos_y - last_pos_y;
-                if py > c2.0 as isize {
-                    return;
-                }
-                
-                // Draw the grapheme to the screen if it's in bounds
-                if (px >= c1.1 as isize) && (py >= c1.0 as isize) && (px <= c2.1 as isize) {
-                    // Check if the character is within a cursor
-                    let mut at_cursor = false;
-                    for c in editor.cursors.iter() {
-                        if grapheme_index >= c.range.0 && grapheme_index <= c.range.1 {
-                            at_cursor = true;
                         }
+                        last_pos_y = pos_y;
                     }
-                
+                    // Calculate the cell coordinates at which to draw the grapheme
-                    // Actually print the character
+                    let px = pos_x as isize + screen_col - editor.view_pos.1 as isize;
-                    if is_line_ending(g) {
+                    let py = lines_traversed as isize + screen_line;
-                        if at_cursor {
+                    
-                            self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, " ");
+                    // If we're off the bottom, we're done
-                        }
+                    if py > c2.0 as isize {
+                        return;
                     }
-                    else if g == "\t" {
+                    
-                        for i in 0..width {
+                    // Draw the grapheme to the screen if it's in bounds
-                            let tpx = px as usize + i;
+                    if (px >= c1.1 as isize) && (py >= c1.0 as isize) && (px <= c2.1 as isize) {
-                            if tpx <= c2.1 {
+                        // Check if the character is within a cursor
-                                self.rb.print(tpx as usize, py as usize, rustbox::RB_NORMAL, Color::White, Color::Black, " ");
+                        let mut at_cursor = false;
+                        for c in editor.cursors.iter() {
+                            if grapheme_index >= c.range.0 && grapheme_index <= c.range.1 {
+                                at_cursor = true;
                             }
                         }
-                        
+                    
-                        if at_cursor {
+                        // Actually print the character
-                            self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, " ");
+                        if is_line_ending(g) {
+                            if at_cursor {
+                                self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, " ");
+                            }
                         }
-                    }
+                        else if g == "\t" {
-                    else {
+                            for i in 0..width {
-                        if at_cursor {
+                                let tpx = px as usize + i;
-                            self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, g);
+                                if tpx <= c2.1 {
+                                    self.rb.print(tpx as usize, py as usize, rustbox::RB_NORMAL, Color::White, Color::Black, " ");
+                                }
+                            }
+                            
+                            if at_cursor {
+                                self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, " ");
+                            }
                         }
                         else {
-                            self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::White, Color::Black, g);
+                            if at_cursor {
+                                self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::Black, Color::White, g);
+                            }
+                            else {
+                                self.rb.print(px as usize, py as usize, rustbox::RB_NORMAL, Color::White, Color::Black, g);
+                            }
                         }
                     }
                 }
+                else {
+                    break;
+                }
                 
                 grapheme_index += 1;
+                line_g_index += 1;
+                
+                if line_g_index >= LINE_BLOCK_LENGTH {
+                    line_block_index += 1;
+                    line_g_index = 0;
+                    g_iter = editor.formatter.iter(line.grapheme_iter_at_index(line_block_index * LINE_BLOCK_LENGTH));
+                    lines_traversed += 1;
+                }
             }
             
-            screen_line += last_pos_y as isize + 1; 
+            screen_line += lines_traversed as isize + 1; 
             line_num += 1;
         }
         
@@ -455,7 +477,7 @@ impl TermUI {
         if at_cursor {
             // Calculate the cell coordinates at which to draw the cursor
             let line = self.editor.buffer.get_line(self.editor.buffer.line_count()-1);
-            let (_, pos_x) = editor.formatter.index_to_v2d(line, line.grapheme_count());
+            let (_, pos_x) = editor.formatter.index_to_v2d(line.grapheme_iter(), line.grapheme_count());
             let px = pos_x as isize + screen_col - editor.view_pos.1 as isize;
             let py = screen_line - 1;
             

todo.md

@@ -15,6 +15,7 @@
     - Loading/saving code for different encodings.
     - Auto-detecting text encodings from file data (this one will be tricky).
 
+- Handle extremely long lines well.
 - Word wrap.
 - Get non-wrapping text working again.
 - File opening by entering path