Merge branch 'master' into new_mom

bin/pi_qemu.sh

@@ -1 +1 @@
-qemu-system-arm -kernel pi/kernel-qemu-4.9.59-stretch -dtb pi/versatile-pb.dtb -cpu arm1176 -m 256 -M versatilepb -no-reboot -serial stdio -append 'root=/dev/sda2 panic=1 rootfstype=ext4 rw' -hda pi/raspbian-stretch-lite.qcow -net nic -net user,hostfwd=tcp::2222-:22
+qemu-system-arm -kernel pi/kernel-qemu-4.19.50-buster -dtb pi/versatile-pb.dtb -cpu arm1176 -m 256 -M versatilepb -no-reboot -serial stdio -append 'root=/dev/sda2 panic=1 rootfstype=ext4 rw' -hda pi/raspbian-buster-lite.qcow -net nic -net user,hostfwd=tcp::2222-:22

bin/rubyxc

@@ -1,4 +1,4 @@
-#! /usr/bin/env ruby -I lib
+#!/usr/bin/env -S ruby -I lib
 require 'rubygems'
 require 'bundler'
 begin

bin/sync_linux.sh

@@ -0,0 +1,2 @@
+rsync -r -a -v --exclude ".git" --exclude "pi" --exclude "vendor" --exclude ".bundle" -e "ssh -l pi -p 2222" /home/torsten/ruby-x/rubyx localhost:/home/pi/
+#afplay /System/Library/Sounds/Morse.aiff

lib/parfait/factory.rb

@@ -50,16 +50,15 @@
     # and rebuilt the reserve (get_next already instantiates the reserve)
     #
     def get_more
-      first_object = get_chain
+      self.reserve = get_chain
-      link = first_object
+      last_link = self.reserve
       count = Factory.reserve_size
       while(count > 0)
-        link = get_next_for(link)
+        last_link = get_next_for(last_link)
         count -= 1
       end
-      self.next_object = get_next_for(link)
+      self.next_object = get_next_for(last_link)
-      set_next_for( link , nil )
+      set_next_for( last_link , nil )
-      self.reserve = first_object
       self
     end
 

lib/risc/collector.rb

@@ -1,43 +1,76 @@
 module Risc
 
-  #  collect anything that is in the space but and reachable from init
+  #  collect anything that is in the space and reachable (linker constants)
   #
   # The place we collect in is the position map in Position class
   module Collector
+    # Collect all object that need to be added to the binary
+    # This means the object_space and aby constants the linker has
+    # we call keep on each object, see there for details
+    # return all positions
     def self.collect_space(linker)
-      keep Parfait.object_space , 0
+      keep Parfait.object_space
       linker.constants.each do |obj|
-        keep(obj,0)
+        keep(obj)
       end
       Position.positions
     end
 
-    def self.keep( object , depth )
+    # keep "collects" the object for "keeping". Such objects get written to binary
+    # keeping used to be done by adding to a hash, but now the object is
+    # given a position, and the Position class has a hash of all positions
+    # (the same hash has all objects, off course)
+    def self.keep( object)
+      collection = []
+      mark_1k( object , 0 , collection)
+      collection.each do |obj|
+        #puts "obj  #{obj.object_id}"
+        keep(obj)
+      end
+    end
+
+    # marking object that make up the binary.
+    # "Only" up to 1k stack depth, collect object that make up the "border"
+    #
+    # Collection is an empty arry that is passed on. Objects below 1k get added
+    # So basically it "should" be a return, but then we would keep creating and adding
+    # arrays, most of which would be empty
+    def self.mark_1k(object , depth , collection)
       return if object.nil?
-      return unless add_object( object , depth )
+      if depth > 1000
+        collection << object
+        return
+      end
+      return unless position!( object )
       return unless object.respond_to? :has_type?
       type = object.get_type
-      keep(type  , depth + 1)
+      mark_1k(type  , depth + 1 , collection)
       return if object.is_a? Symbol
       type.names.each do |name|
-        keep(name , depth + 1)
+        mark_1k(name , depth + 1, collection)
         inst = object.get_instance_variable name
-        keep(inst , depth + 1)
+        #puts "getting name #{name}, val=#{inst} #{inst.object_id}"
+        mark_1k(inst , depth + 1, collection)
       end
       if object.is_a? Parfait::List
         object.each do |item|
-          keep(item , depth + 1)
+          mark_1k(item , depth + 1, collection)
         end
       end
     end
 
-    # Objects are data and get assembled after functions
+    # Give the object a position. Position class keeps a list of all positions
-    def self.add_object( objekt , depth)
+    # and associated objects. The actual position is determined later, here a
+    # Position object is assigned.
+    #
+    # All Objects that end up in the binary must have a Position.
+    #
+    # return if the position was assigned (true) or had been assigned already (false)
+    def self.position!( objekt )
       return false if Position.set?(objekt)
       return true if objekt.is_a? ::Integer
       return true if objekt.is_a?( Risc::Label)
-      #puts message(objekt , depth)
+      #puts "ADD  #{objekt.class.name}"
-      #puts "ADD #{objekt.inspect}, #{objekt.name}" if objekt.is_a? Parfait::CallableMethod
       unless objekt.is_a?( Parfait::Object) or objekt.is_a?( Symbol)
         raise "adding non parfait #{objekt.class}:#{objekt}"
       end
@@ -46,13 +79,5 @@ module Risc
       true
     end
 
-    def self.message(object , depth)
-      msg =  "adding #{depth}#{' ' * depth}:"
-      if( object.respond_to?(:rxf_reference_name))
-        msg + object.rxf_reference_name.to_s
-      else
-        msg + object.class.name
-      end
-    end
   end
 end

lib/risc/interpreter.rb

@@ -8,7 +8,8 @@ module Risc
   # will be executed by method execute_SlotToReg
   #
   # The Interpreter (a bit like a cpu) has a state flag, a current instruction and registers
-  # We collect the stdout (as a hack not to interpret the OS)
+  # We collect the stdout (as a hack not to interpret the OS) in a string. It can also be passed
+  # in to the init, as an IO
   #
   class Interpreter
     # fire events for changed pc and register contents
@@ -18,11 +19,13 @@ module Risc
 
     attr_reader :instruction , :clock , :pc  # current instruction and pc
     attr_reader :registers     # the registers, 16 (a hash, sym -> contents)
-    attr_reader :stdout,  :state , :flags  # somewhat like the lags on a cpu, hash  sym => bool (zero .. . )
+    attr_reader :stdout,  :state , :flags  # somewhat like the flags on a cpu, hash  sym => bool (zero .. . )
 
-    #start in state :stopped and set registers to unknown
+    # start in state :stopped and set registers to unknown
-    def initialize( linker )
+    # Linker gives the state of the program
-      @stdout , @clock , @pc , @state = "", 0 , 0 , :stopped
+    # Passing a stdout in (an IO, only << called) can be used to get output immediately.
+    def initialize( linker , stdout = "")
+      @stdout , @clock , @pc , @state = stdout, 0 , 0 , :stopped
       @registers = {}
       @flags = {  :zero => false , :plus => false ,
                   :minus => false , :overflow => false }
@@ -32,8 +35,7 @@ module Risc
       @linker = linker
     end
 
-    def start_program(linker = nil)
+    def start_program()
-      initialize(linker || @linker)
       init = @linker.cpu_init
       set_state(:running)
       set_pc( Position.get(init).at )
@@ -249,10 +251,12 @@ module Risc
       str = get_register( :r1 ) # should test length, ie r2
       case str
       when Symbol
-        @stdout += str.to_s
+        @stdout << str.to_s
+        @stdout.flush if @stdout.respond_to?(:flush)
         return str.to_s.length
       when Parfait::Word
-        @stdout += str.to_string
+        @stdout << str.to_string
+        @stdout.flush if @stdout.respond_to?(:flush)
         return str.char_length
       else
         raise "NO string for putstring #{str.class}:#{str.object_id}" unless str.is_a?(Symbol)

lib/risc/parfait_boot.rb

@@ -152,8 +152,10 @@ module Parfait
       Data8: {},
       Data16: {},
       Dictionary: {i_keys: :List , i_values: :List  } ,
-      Integer: {next_integer: :Integer},
       FalseClass: {},
+      Factory: { for_type: :Type , next_object: :Object ,
+                 reserve: :Object , attribute_name: :Word },
+      Integer: {next_integer: :Integer},
       List: {indexed_length: :Integer , next_list: :List} ,
       Message: { next_message: :Message,   receiver: :Object, frame: :NamedList ,
                  return_address: :Integer, return_value: :Object,
@@ -162,8 +164,6 @@ module Parfait
       NamedList: {},
       NilClass: {},
       Object: {},
-      Factory: { for_type: :Type , next_object: :Object ,
-              reserve: :Object , attribute_name: :Word },
       ReturnAddress: {next_integer: :ReturnAddress},
       Space: {classes: :Dictionary , types: :Dictionary , factories: :Dictionary,
               true_object: :TrueClass, false_object: :FalseClass , nil_object: :NilClass},

lib/rubyx/rubyxc.rb

@@ -1,11 +1,14 @@
 require "thor"
 require "rubyx"
+require "risc/interpreter"
 
 class RubyXC < Thor
+  class_option :parfait , type: :numeric
+
+
   desc "compile FILE" , "Compile given FILE to binary"
   long_desc <<-LONGDESC
-      Very basic cli to compile ruby programs.
+      Compile the give file name to binary object file (see long descr.)
-      Currently only compile command supported without option.
 
       Output will be elf object file of the same name, with .o, in root directory.
 
@@ -22,7 +25,7 @@ class RubyXC < Thor
     end
     puts "compiling #{file}"
 
-    linker = ::RubyX::RubyXCompiler.new({}).ruby_to_binary( ruby , :arm )
+    linker = ::RubyX::RubyXCompiler.new(extract_options).ruby_to_binary( ruby , :arm )
     writer = Elf::ObjectWriter.new(linker)
 
     outfile = file.split("/").last.gsub(".rb" , ".o")
@@ -30,4 +33,74 @@ class RubyXC < Thor
 
     return outfile
   end
+
+
+  desc "interpret FILE" , "Interpret given FILE "
+  long_desc <<-LONGDESC
+  Compiles the given file to an intermediate RISC format, and runs the
+  Interpreter.
+
+  RISC is the last abstract layer inside the compiler. It is in nature
+  very close to arm (without quirks and much smaller).
+
+  An interpreter was originally developed for the RISC layer for debugging purposes.
+  Running the interpreter is about 50k slower than binary, but it can be used
+  to veryfy simple programs.
+
+  No output file will be generated, the only output is generated by the
+  given program.
+
+  The program must define a main method on the Space class, which will be invoked.
+  LONGDESC
+
+  def interpret(file)
+    begin
+      ruby = File.read(file)
+    rescue
+      fail MalformattedArgumentError , "No such file #{file}"
+    end
+    compiler = RubyX::RubyXCompiler.new(extract_options)
+    linker = compiler.ruby_to_binary(ruby, :interpreter)
+
+    puts "interpreting #{file}"
+
+    interpreter = Risc::Interpreter.new(linker , STDOUT )
+    interpreter.start_program
+    interpreter.tick while(interpreter.instruction)
+
+  end
+
+  desc "execute FILE" , "Compile given FILE and execute resulting binary"
+  long_desc <<-LONGDESC
+    Just like the compile task, this compiles the file to an object/binary file.
+
+    Then rubyxc will link and run the resulting object file. For this to work,
+    qemu needs to be set up correctly on the system. Specifically, because of
+    bug #13, arm-linux-gnueabihf-ld needs to exist (it's part of the cross compiled
+    arm binutils).
+
+    The resulting a.out will be run via qemu-arm. This is part of the qemu "linux" package
+    and interprets the arm binary on the host, assuming a linux os.
+
+    This whole approach should only be used for preliminary checking that no core-dumps
+    are generated by the program, or when no benchmarking (as the times will be whatever).
+
+    For simple functional test though, it is a much much quicker way to run the binary
+    than transferring it to another machine. The a.out is left in place to be run again.
+  LONGDESC
+
+  def execute(file)
+    outfile = compile(file)
+    system "arm-linux-gnueabihf-ld -N #{outfile}"
+    puts "Linked ok, now running #{file}"
+    system "qemu-arm ./a.out"
+  end
+
+  private
+  def extract_options
+    opt = { factory: options[:parfait] || 1024 }
+    puts opt
+    return {parfait: opt}
+  end
+
 end

test/bench/c/adds.c

@@ -1,5 +1,3 @@
-#include<stdio.h>
-
 int fibo(int n){
   int result;
 	int a = 0;
@@ -17,11 +15,10 @@ int fibo(int n){
 
 int main(void)
 {
-	int counter = 100352 - 352;
+  int counter = 50000;
-	int counter2 = counter;
-	int level = 40;
 	int fib ;
-	while(counter--) {
+	while(counter) {
-		fib = fibo(level);
+		fib = fibo(40);
+    counter -= 1;
 	}
 }

test/bench/c/calls.c

@@ -10,10 +10,9 @@ int fibo_r(int n)
 
 int main(void)
 {
-	int counter = 1000;
+	int counter = 100;
-	int counter2 = counter;
 	int fib ;
 	while(counter--) {
-		fib = fibo_r(20);
+		fib += fibo_r(20);
 	}
 }

test/bench/c/hello.c

@@ -2,8 +2,8 @@
 
 int main(void)
 {
-	setbuf(stdout, NULL); /* to make it equivalent to the typed version, otherwise it caches */
+  setbuf(stdout, NULL); /* to make it equivalent to the other versions, otherwise it caches */
-	int counter = 100352 - 352;
+  int counter = 10000;
 	while(counter--) {
     printf("Hello there\n");
 	}

test/bench/c/itos.c

@@ -3,9 +3,9 @@
 int main(void)
 {
 	char stringa[20] ;
-
+	int counter = 1000;
-	int counter = 100352 - 352;
 	while(counter--) {
 		sprintf(stringa, "%i\n" , counter);
 	}
+
 }

test/bench/c/loop.c

@@ -1,9 +1,7 @@
-#include<stdio.h>
-
 int main(void)
 {
-int counter = 100352 - 352;
+  int counter = 1000000;
 	while(counter) {
-    counter = counter - 1;
+    counter -= 1;
 	}
 }

test/bench/c/noop.c

@@ -0,0 +1,5 @@
+
+int main(void)
+{
+  return 0;
+}

test/bench/go/adds.go

@@ -18,8 +18,10 @@ func fibo(n int ) int {
 
 func main() {
   sum := 1
-  for sum < 100000 {
+  res := 0
+  for sum < 50000 {
     sum += 1
-    fibo( 40 )
+    res = fibo( 40 )
   }
+  res += 1
 }

test/bench/go/calls.go

@@ -10,7 +10,7 @@ func fib(n uint) uint {
 
 func main() {
   sum := 1
-  for sum < 1000 {
+  for sum < 100 {
     sum += 1
     fib( 20 )
   }

test/bench/go/hello.go

@@ -4,7 +4,7 @@ import "fmt"
 
 func main() {
 	sum := 1
-	for sum < 100000 {
+	for sum < 10000 {
 		sum += 1
     fmt.Println("Hi there")
 	}

test/bench/go/loop.go

@@ -2,7 +2,7 @@ package main
 
 func main() {
 	sum := 1
-	for sum < 100000 {
+	for sum < 1000000 {
 		sum += 1
 	}
 }

test/bench/go/noop.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+  return 
+}

test/bench/results.md

@@ -1,36 +1,30 @@
 # Benchmarks
 
-loop  - program does empty loop of same size as hello
+hello - output hello world to measure kernel calls
-hello - output hello world (to dev/null) to measure kernel calls (not terminal speed)
-itos  - convert integers from 1 to 100000  to string
 add   - run integer adds by linear fibonacci of 40
 call  - exercise calling by recursive fibonacci of 20
+noop  - a baseline that does nothing
+loop  -  just counts down, from 1M
 
-Hello and puti and add run 100_000 iterations per program invocation to remove startup overhead.
+Loop, Hello, add and call run 1M , 50k, 10k and 100 respectively,
-Call only has 10000 iterations, as it much slower
+to minimize startup impact.
 
-Gcc used to compile c on the machine
+C was linked statically as dynamic linked influences times.
-typed produced by ruby (on another machine)
+Output was sent to /dev/null, so as to measure the calling and not the terminal.
+Also output was unbuffered, because that is what rubyx implements.
 
 # Results
 
 Results were measured by a ruby script. Mean and variance was measured until variance was low,
-always under one percent.
+always under one percent. Noop showed that program startup is a factor, so all programs loop somewhere from 1M to 100, depending on how intensive.
 
-The machine was a virtual arm run on a powerbook, performance roughly equivalent to a raspberry pi.
+The machine was a virtual arm (qemu) run on a acer swift 5 (i5 8265 3.9GHz), performance roughly equivalent to a raspberry pi.
-But results should be seen as relative, not absolute.
+Results (in ms) should be seen as relative, not absolute.
 
 
-language  | loop    | hello   | itos    |  add    | call        
+language  |  noop   |  hello   |  add   |  call | loop        
-c         | 0,0500  | 2,1365  | 0,2902  | 0,1245  | 0,8535  
+c         |    55   |   380    |   88   |   135 |    6
-go        | 0.0485  | 4.5355  | 0.2143  | 0.0825  | 0.8769  
+go        |    52   |   450    |    9   |    77 |    2
-typed     | 0,0374  | 1,2071  | 0,7263  | 0,2247  | 1,3625
+rubyx     |    42   |   200    | 1700   |  1700 |  480
-
+ruby      |  1570   |   650    | 1090   |  1500 |  180
-ruby      | 0,3     | 8.882    | 0,8971 | 3,8452
+mruby     |    86   |  1200    | 1370   |  2700 |  300
-
-2c        | - 33 %  | - 79 %  | + 150%  | + 80 %  | + 60 %
-
-2r        | x 10    | x 6     | + 23%   | x 17    | x 26
-
-Comparison with ruby, not really for speed, just to see how much leeway there is for our next layer.
-Ruby startup time is 1,5695 seconds, which we'll subtract from the benches

test/bench/ruby/adds.rb

@@ -11,7 +11,7 @@ def fibo( n)
 	return result
 end
 
- counter = 100000
+ counter = 50000
 
 while(counter > 0) do
 	fibo(40)

test/bench/ruby/calls.rb

@@ -8,7 +8,7 @@ def fibo_r( n )
 end
 
 
- counter = 1000
+ counter = 100
 
  while(counter > 0) do
    fibo_r(20)

test/bench/ruby/hello.rb

@@ -1,7 +1,8 @@
 
-counter = 100352 - 352;
+counter = 10000;
 while(counter > 0) do
   puts "Hello there"
+  # roughly 4 times slower with this, which is like rubyx
   STDOUT.flush
   counter = counter - 1
 end

test/bench/ruby/loop.rb

@@ -1,5 +1,5 @@
 
-counter = 100000
+counter = 1000000
 while(counter > 0) do
-  counter = counter - 1
+  counter -= 1
 end

test/bench/ruby/noop.rb

@@ -0,0 +1 @@
+return 0

test/bench/rubyx/adds.rb

@@ -0,0 +1,26 @@
+class Space
+
+  def fibo_i(fib)
+    n = fib
+    a = 0
+    b = 1
+    i = 1
+    while( i < n )
+      result = a + b
+      a = b
+      b = result
+      i = i + 1
+    end
+    return result
+  end
+
+  # ran with --parfait=80000
+  def main(arg)
+    b = 1000
+    while( b >= 1 )
+      b = b - 1
+      fibo_i(40)
+    end
+    return b
+  end
+end

test/bench/rubyx/calls.rb

@@ -0,0 +1,22 @@
+class Space
+
+  def fibo_r( n )
+     if( n <  2 )
+       return n
+     end
+     a = fibo_r(n - 1)
+     b = fibo_r(n - 2)
+     return a + b
+  end
+
+  # ran with --parfait=70000
+  def main(arg)
+    b = 2
+    res = 0
+    while( b >= 1 )
+      b = b - 1
+      res = fibo_r(20)
+    end
+    return res
+  end
+end

test/bench/rubyx/hello.rb

@@ -0,0 +1,11 @@
+class Space
+  # ran with --parfait=25000
+  def main(arg)
+    b = 10000
+    while( b >= 1 )
+      b = b - 1
+      "Hello-there\n".putstring
+    end
+    return b
+  end
+end

test/bench/rubyx/loop.rb

@@ -0,0 +1,11 @@
+class Space
+
+  # ran with --parfait=101000
+  def main(arg)
+    b = 100000
+    while( b >= 1 )
+      b = b - 1
+    end
+    return b
+  end
+end

test/bench/rubyx/noop.rb

@@ -0,0 +1,5 @@
+class Space
+  def main(arg)
+    return 0
+  end
+end

test/bench/runner.rb

@@ -17,7 +17,7 @@ class Stats
   def show
     #puts "no    per   var"
 
-    puts "#{@n}    #{@mean}   #{@variance / @n}"
+    puts "#{@n}    #{(@mean*1000).truncate(1)}   #{((@variance / @n)*100).truncate(2)}"
   end
 end
 class Runner

test/mains/source/fibo__8.rb

@@ -13,3 +13,4 @@ class Space
     return result
   end
 end
+#Space.new.main(1)

test/risc/test_collector.rb

@@ -11,7 +11,7 @@ module Risc
 
     def test_simple_collect
       objects = Collector.collect_space(@linker)
-      assert ((400 < objects.length) or (450 > objects.length)) , objects.length.to_s
+      assert_equal 600 ,  objects.length , objects.length.to_s
     end
 
     def test_collect_all_types
@@ -34,5 +34,40 @@ module Risc
         assert !position.valid?
       end
     end
+    def test_integer_positions
+      objects = Collector.collect_space(@linker)
+      int = Parfait.object_space.get_next_for(:Integer)
+      while(int)
+        assert Position.set?(int) , "INt #{int.object_id}"
+        int = int.next_integer
+      end
+    end
+  end
+  class TestBigCollector < MiniTest::Test
+
+    def setup
+      opt = Parfait.default_test_options
+      opt[:factory] = 4000
+      Parfait.boot!(opt)
+      Risc.boot!
+      @linker = Mom::MomCompiler.new.translate(:arm)
+    end
+
+    def test_simple_collect
+      objects = Collector.collect_space(@linker)
+      assert_equal 20329, objects.length , objects.length.to_s
+    end
+
+    def test_integer_positions
+      objects = Collector.collect_space(@linker)
+      int = Parfait.object_space.get_next_for(:Integer)
+      count = 0
+      while(int)
+        count += 1
+        assert Position.set?(int) , "INT #{int.object_id} , count #{count}"
+        int = int.next_integer
+      end
+    end
+
   end
 end