Gecko [ fernandoquevedodop.com ]

Name	Size	Permission
deferrable	[ DIR ]	drwxr-xr-x
protocols	[ DIR ]	drwxr-xr-x
buftok.rb	2.13 KB	-rw-r--r--
callback.rb	2.14 KB	-rw-r--r--
channel.rb	1.53 KB	-rw-r--r--
completion.rb	9.71 KB	-rw-r--r--
connection.rb	33.41 KB	-rw-r--r--
deferrable.rb	7.8 KB	-rw-r--r--
file_watch.rb	2 KB	-rw-r--r--
future.rb	1.83 KB	-rw-r--r--
io_streamer.rb	1.86 KB	-rw-r--r--
iterator.rb	7.04 KB	-rw-r--r--
messages.rb	2.77 KB	-rw-r--r--
pool.rb	4.1 KB	-rw-r--r--
process_watch.rb	1.24 KB	-rw-r--r--
processes.rb	3.66 KB	-rw-r--r--
protocols.rb	1.5 KB	-rw-r--r--
pure_ruby.rb	34.87 KB	-rw-r--r--
queue.rb	2.06 KB	-rw-r--r--
resolver.rb	4.76 KB	-rw-r--r--
spawnable.rb	2.12 KB	-rw-r--r--
streamer.rb	3.59 KB	-rw-r--r--
threaded_resource.rb	2.8 KB	-rw-r--r--
tick_loop.rb	2.09 KB	-rw-r--r--
timers.rb	1.29 KB	-rw-r--r--
version.rb	50 B	-rw-r--r--

Code Editor : buftok.rb

# BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
# by default.  It allows input to be spoon-fed from some outside source which
# receives arbitrary length datagrams which may-or-may-not contain the token
# by which entities are delimited.  In this respect it's ideally paired with
# something like EventMachine (http://rubyeventmachine.com/).
class BufferedTokenizer
  # New BufferedTokenizers will operate on lines delimited by a delimiter,
  # which is by default the global input delimiter $/ ("\n").
  #
  # The input buffer is stored as an array.  This is by far the most efficient
  # approach given language constraints (in C a linked list would be a more
  # appropriate data structure).  Segments of input data are stored in a list
  # which is only joined when a token is reached, substantially reducing the
  # number of objects required for the operation.
  def initialize(delimiter = $/)
    @delimiter = delimiter
    @input = []
    @tail = ''
    @trim = @delimiter.length - 1
  end

# Extract takes an arbitrary string of input data and returns an array of
  # tokenized entities, provided there were any available to extract.  This
  # makes for easy processing of datagrams using a pattern like:
  #
  #   tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
  #
  # Using -1 makes split to return "" if the token is at the end of
  # the string, meaning the last element is the start of the next chunk.
  def extract(data)
    if @trim > 0
      tail_end = @tail.slice!(-@trim, @trim) # returns nil if string is too short
      data = tail_end + data if tail_end
    end

@input << @tail
    entities = data.split(@delimiter, -1)
    @tail = entities.shift

unless entities.empty?
      @input << @tail
      entities.unshift @input.join
      @input.clear
      @tail = entities.pop
    end

entities
  end

# Flush the contents of the input buffer, i.e. return the input buffer even though
  # a token has not yet been encountered
  def flush
    @input << @tail
    buffer = @input.join
    @input.clear
    @tail = "" # @tail.clear is slightly faster, but not supported on 1.8.7
    buffer
  end
end

${this.title}

Code Editor : buftok.rb