In my previous post I explained how to compile and link Rust modules for the x86 target, showing how to print a simple message from rust to the xv6 console.

In this guide I will expand on the framework explained above, implementing interrupt-driven networking with a single TCP socket (similar to *BSD) in the xv6 kernel, fully written in Rust, using the Intel e1000 network card.

On top of this, this project is built on the public xv6 source (instead of my school’s private xv6 fork) meaning that I can show the full repo.

I will then show this functionality by telling ChatGPT 5.1 what syscalls it has access to and making it write a userspace HTTP server.

e1000 xv6 Driver layer

There are a lot of great guides explaining how the e1000 works and how to interface to it in xv6.

On a very basic level:

• You first initialize the card by sending it a series of numbers over PCI.
  • On the internet you’ll find very extensive docs explaining exactly what everything does, but ChatGPT does a good job at making it so that you don’t have to read through function codes.
• When the e1000 NIC receives a packet, it writes it to a circular buffer and generates an interrupt
• When you want to transmit a packet, you write a packet to a separate circular buffer and set a flag.

The kernel should therefore map these buffers into memory and then use MMIO to read and write to the network card.

From the docs linked above

How I chose to implement this was by having four main functions:

1. e1000_init(void)
   • Reads card, maps memory, sends codes to set the card in the right mode.
2. e1000_tx(data, len)
   • Given ethernet frames (and its length), write packets to the circular buffer.
3. e1000_rx_poll(buf, len)
   • Check the NIC RX ring, if we received a packet, read it into buf.
4. e1000_intr(void)
   • The interrupt handler, when we receive an interrupt, pass control to the Rust handler.

Init e1000

void
e1000_init(void)
{
  ioapicenable(IRQ_E1000, 0);
  uint bus = 0;
  uint slot = 3;

  uint bar0 = pci_config_read32(bus, slot, 0, 0x10);
  uint mmio_pa = bar0 & ~0xF;

  uint pa_page = mmio_pa & ~(PGSIZE - 1);
  void *va_page = (void*)P2V(pa_page);
  uint mmio_size = 0x4000; // 4 pages

  int perm = PTE_W | PTE_P;

  if(mappages(kpgdir, va_page, mmio_size, pa_page, perm) < 0){
    cprintf("e1000: mappages failed for mmio_pa=0x%x\n", mmio_pa);
    return;
  }

  // Reload CR3 so CPU picks up new mapping
  lcr3(V2P(kpgdir));

  e1000_regs = (volatile uint *)P2V(mmio_pa);
  cprintf("e1000: BAR0=0x%x mmio_pa=0x%x regs=%p\n", bar0, mmio_pa, e1000_regs);

  uint status = e1000_read_reg(E1000_STATUS);
  cprintf("e1000: status=0x%x\n", status);

  // Enable bus mastering + memory space on the PCI device
  uint cmd = pci_config_read32(bus, slot, 0, 0x04);
  ushort cmd_lo = cmd & 0xFFFF;

  // Bit 1 = Memory Space Enable, Bit 2 = Bus Master Enable
  cmd_lo |= 0x0002; // MSE
  cmd_lo |= 0x0004; // BME

  cmd = (cmd & 0xFFFF0000) | cmd_lo;
  pci_config_write32(bus, slot, 0, 0x04, cmd);

  cprintf("e1000: PCI CMD=0x%x\n", cmd_lo);

  // Init TX ring
  int i;
  for(i = 0; i < TX_RING_SIZE; i++){
    tx_ring[i].addr_lo = V2P(tx_bufs[i]);
    tx_ring[i].addr_hi = 0;
    tx_ring[i].length  = 0;
    tx_ring[i].cso     = 0;
    tx_ring[i].cmd     = E1000_TXD_CMD_RS | E1000_TXD_CMD_IFCS;
    tx_ring[i].status  = E1000_TXD_STAT_DD;  // mark free
    tx_ring[i].css     = 0;
    tx_ring[i].special = 0;
  }

  e1000_write_reg(E1000_TDBAL, V2P(tx_ring));
  e1000_write_reg(E1000_TDBAH, 0);
  e1000_write_reg(E1000_TDLEN, sizeof(tx_ring));
  e1000_write_reg(E1000_TDH, 0);
  e1000_write_reg(E1000_TDT, 0);
  tx_tail = 0;

  // Init RX ring
  for(i = 0; i < RX_RING_SIZE; i++){
    rx_ring[i].addr_lo = V2P(rx_bufs[i]);
    rx_ring[i].addr_hi = 0;
    rx_ring[i].length  = 0;
    rx_ring[i].csum    = 0;
    rx_ring[i].status  = 0;   // owned by NIC
    rx_ring[i].errors  = 0;
    rx_ring[i].special = 0;
  }

  e1000_write_reg(E1000_RDBAL, V2P(rx_ring));
  e1000_write_reg(E1000_RDBAH, 0);
  e1000_write_reg(E1000_RDLEN, sizeof(rx_ring));
  e1000_write_reg(E1000_RDH, 0);
  e1000_write_reg(E1000_RDT, RX_RING_SIZE - 1);
  rx_next = 0;

  // 5) Configure transmit control
  uint tctl = 0;
  tctl |= E1000_TCTL_EN;   // enable TX
  tctl |= E1000_TCTL_PSP;  // pad short packets
  tctl |= (0x10 << E1000_TCTL_CT_SHIFT);   // collision threshold ~16
  tctl |= (0x40 << E1000_TCTL_COLD_SHIFT); // collision distance ~64
  e1000_write_reg(E1000_TCTL, tctl);

  // 6) Configure inter-packet gap
  uint tipg = 0;
  tipg |= E1000_TIPG_IPGT;
  tipg |= (E1000_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT);
  tipg |= (E1000_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT);
  e1000_write_reg(E1000_TIPG, tipg);

  // 7) Configure receive control
  uint rctl = 0;
  rctl |= E1000_RCTL_EN;        // enable RX
  rctl |= E1000_RCTL_BAM;       // accept broadcast
  rctl |= E1000_RCTL_UPE;       // accept all unicast
  rctl |= E1000_RCTL_MPE;       // accept all multicast
  rctl |= E1000_RCTL_SZ_2048;   // 2K buffers
  rctl |= E1000_RCTL_SECRC;     // strip CRC
  e1000_write_reg(E1000_RCTL, rctl);

  e1000_write_reg(E1000_IMC, 0xffffffff);  // mask all
  (void)e1000_read_reg(E1000_ICR);         // clear pending
  e1000_write_reg(E1000_IMS,
                  E1000_IMS_RXT0 |   // RX timer / normal receive
                  E1000_IMS_RXO  |   // RX overrun (optional)
                  E1000_IMS_RXDMT0 | // RX desc low watermark
                  E1000_IMS_TXDW);   // TX descriptor write-back

  // debug: read back RCTL to confirm
  uint rctl_read = e1000_read_reg(E1000_RCTL);
  cprintf("e1000: RCTL=0x%x\n", rctl_read);

  (void)e1000_test_send;
  // e1000_test_send();  // optional

  cprintf("e1000: initialized\n");

  // Initialize Rust+smoltcp stack now that NIC is ready.
  rust_net_init();
}

Transmit frame

int
e1000_tx(const void *data, int len)
{
  if(len <= 0 || len > PKT_BUF_SIZE)
    return -1;

  int tdt = e1000_read_reg(E1000_TDT);
  struct e1000_tx_desc *d = &tx_ring[tdt];

  cprintf("e1000_tx: tdt=%d status=0x%x\n", tdt, d->status);

  if((d->status & E1000_TXD_STAT_DD) == 0){
    cprintf("e1000_tx: ring full at %d\n", tdt);
    return -1;
  }

  memmove(tx_bufs[tdt], data, len);

  d->length = len;
  d->status = 0;
  d->cmd = E1000_TXD_CMD_RS | E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;

  tdt = (tdt + 1) % TX_RING_SIZE;
  e1000_write_reg(E1000_TDT, tdt);

  cprintf("e1000_tx: queued len=%d new TDT=%d\n", len, tdt);
  return 0;
}

Read frame

int
e1000_rx_poll(uchar *buf, int maxlen)
{
  struct e1000_rx_desc *d = &rx_ring[rx_next];

  if((d->status & E1000_RXD_STAT_DD) == 0)
    return -1;

  int n = 0;

  if((d->status & E1000_RXD_STAT_EOP) == 0){
    cprintf("e1000: RX fragment, dropping (len=%d status=0x%x)\n",
                  d->length, d->status);
    // n stays 0 -> caller won't use this packet
    goto rearm;
  }

  n = d->length;
  if(n > maxlen)
    n = maxlen;

  memmove(buf, rx_bufs[rx_next], n);

rearm:
  d->status = 0;
  d->errors = 0;

  rx_next = (rx_next + 1) % RX_RING_SIZE;
  int rdt = (rx_next + RX_RING_SIZE - 1) % RX_RING_SIZE;
  e1000_write_reg(E1000_RDT, rdt);

  return n;
}

Handle Interrupt

void
e1000_intr(void)
{
  uint icr = e1000_read_reg(E1000_ICR);

  if(icr == 0)
    return;

  // RX-related interrupt?
  if(icr & (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)){
    // Let Rust+smoltcp pull packets with e1000_rx_poll() and handle them.
    cprintf("handling packet in rust\n");
    rust_net_poll();
  }

  if(icr & E1000_ICR_TXDW){
    cprintf("e1000: TXDW\n");
  }

  (void)e1000_debug_dump;
}

OS Layer

“Manual” Networking

I started the project by implementing my own TCP/IP stack, however I realized early on that it would have involved writing hundreds of lines of code handling every specific IP protocols that I wanted (ICMP, HTTP, just to name a few) by reading their spec and manually creating all the specific packets, and I have no interest in doing so right now, given that this is an OS-focused project.

/// Inside the interrupt handler
...
    match ethertype {
        0x0806 => {
            // ARP
            unsafe { puts_const(b"rust_net_rx: ARP frame\n\0") };
            arp_rx(payload, src, dst);
        }
        0x0800 => {
            // IPv4
            unsafe { puts_const(b"rust_net_rx: IPv4 frame\n\0") };
            ipv4_rx(payload, src, dst);
        }
        _ => {
            // unsupported ethertype, drop silently for now
        }
...

/// Example of a ~120loc function just to handle basic ARP packets.
/// Most of the funciton is just manually building the correct buffers.
fn arp_rx(payload: &[u8], src_mac: &[u8], _dst_mac: &[u8]) {
    // We assume Ethernet/IPv4 ARP, which is 28 bytes minimum.
    if payload.len() < 28 || src_mac.len() < 6 {
        return;
    }

    let htype = u16::from_be_bytes([payload[0], payload[1]]);
    let ptype = u16::from_be_bytes([payload[2], payload[3]]);
    let hlen  = payload[4];
    let plen  = payload[5];
    let oper  = u16::from_be_bytes([payload[6], payload[7]]);

    // Only handle Ethernet (1) + IPv4 (0x0800), MAC len 6, IP len 4
    if htype != 1 || ptype != 0x0800 || hlen != 6 || plen != 4 {
        return;
    }

    // Layout:
    // [ 8..14 ] sender hw
    // [14..18] sender IP
    // [18..24] target hw
    // [24..28] target IP
    let sha = &payload[8..14];   // sender MAC
    let spa = &payload[14..18];  // sender IP
    let tpa = &payload[24..28];  // target IP
...

Snippet of what the could would have looked like was I to implement everything by myself. Hopefully you can see how this isn’t aligned with the scope of this project.

The smoltcp crate

After some research I came across smoltcp, a “standalone, event-driven TCP/IP stack that is designed for bare-metal, real-time systems”. Very importantly, this crate compiles with no_std, making it an excellent choice for xv6 development.

The simplicity of this crate is what drew me to it. A Device only needs to implement receive, transmit, and capabilities. With these three methods done (which are really only 2, since capabilities is just a way to describe the device’s capabilities.), we basically have a fully functional TCP/IP stack. For my xv6 e1000 driver, these implementations looked like:

impl Device for Xv6Device {
    type RxToken<'a> = Xv6RxToken<'a> where Self: 'a;
    type TxToken<'a> = Xv6TxToken<'a> where Self: 'a;

    fn capabilities(&self) -> DeviceCapabilities {
        let mut caps = DeviceCapabilities::default();
        caps.medium = Medium::Ethernet;
        caps.max_transmission_unit = MAX_FRAME_SIZE as usize;
        caps
    }

    fn receive(
        &mut self,
        _timestamp: Instant,
    ) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> {
        unsafe { puts_const(b"rust: smoltcp receive() got frame\n\0"); }

        self.rx_len = 0;
        // ↓ C function
        let n = unsafe { e1000_rx_poll(self.rx_buf.as_mut_ptr(), MAX_FRAME_SIZE as c_int) };
        if n <= 0 {
            return None;
        }

        self.rx_len = n as usize;
        let rx = Xv6RxToken {
            buffer: &self.rx_buf[..self.rx_len],
        };
        let tx = Xv6TxToken {
            buffer: &mut self.tx_buf[..],
        };
        Some((rx, tx))
    }

    fn transmit(&mut self, _timestamp: Instant) -> Option<Self::TxToken<'_>> {
        unsafe { puts_const(b"rust: smoltcp transmit() called\n\0"); }
        Some(Xv6TxToken {
            buffer: &mut self.tx_buf[..],
        })
    }
}

struct Xv6Device {
    rx_buf: [u8; MAX_FRAME_SIZE],
    rx_len: usize,
    tx_buf: [u8; MAX_FRAME_SIZE],
}

impl Xv6Device {
    const fn new() -> Self {
        Self {
            rx_buf: [0; MAX_FRAME_SIZE],
            rx_len: 0,
            tx_buf: [0; MAX_FRAME_SIZE],
        }
    }
}

struct Xv6RxToken<'a> {
    buffer: &'a [u8],
}

struct Xv6TxToken<'a> {
    buffer: &'a mut [u8],
}

impl<'a> RxToken for Xv6RxToken<'a> {
    fn consume<R, F>(self, f: F) -> R
    where
        F: FnOnce(&[u8]) -> R,
    {
        f(self.buffer)
    }
}

impl<'a> TxToken for Xv6TxToken<'a> {
    fn consume<R, F>(self, len: usize, f: F) -> R
    where
        F: FnOnce(&mut [u8]) -> R,
    {
        let slice = &mut self.buffer[..len];
        let result = f(slice);

        unsafe {
            puts_const(b"rust: smoltcp TxToken::consume, sending frame\n\0");
            // ↓ C function
            e1000_tx(slice.as_ptr(), len as c_int);
        }

        result
    }
}

With this done, we have a working TCP/IP stack that we can use from the xv6 kernel, however the user still has no way to access these resources.

OS Syscall Layer

To give the user access to networking resources, I chose to implement 4 syscalls:

• int net_listen(int port);
  • Listen on TCP port port.
• int net_accept(void);
  • Returns 1 if a connection is established, 0 if not yet, -1 on error.
• int net_recv(void *buf, int n);
  • Returns bytes read, 0 if no data, -1 on error.
• int net_send(void *buf, int n);
  • Returns bytes sent, 0 if would-block, -1 on error.
• int net_close(void);
  • Close current connection.

C Side

On the C side, these functions simply do argument parsing and pass the control to the Rust module:

int
sys_net_listen(void)
{
  int port;
  if(argint(0, &port) < 0)
    return -1;
  if(port < 0 || port > 65535)
    return -1;
  return rust_net_listen(port);
}

// int net_accept(void);
// returns 1 if a connection is established, 0 if not yet, -1 on error
int
sys_net_accept(void)
{
  return rust_net_accept();
}

// int net_recv(void *buf, int len);
int
sys_net_recv(void)
{
  int len;
  char *buf;

  if(argptr(0, &buf, 0) < 0) // size checked below
    return -1;
  if(argint(1, &len) < 0)
    return -1;
  if(len < 0)
    return -1;

  return rust_net_recv(buf, len);
}

// int net_send(void *buf, int len);
int
sys_net_send(void)
{
  int len;
  char *buf;

  if(argptr(0, &buf, 0) < 0)
    return -1;
  if(argint(1, &len) < 0)
    return -1;
  if(len < 0)
    return -1;

  return rust_net_send(buf, len);
}

// int net_close(void);
int
sys_net_close(void)
{
  rust_net_close();
  return 0;
}

Rust Side

The real logic is all implement in the safe Rust module. However, due to smoltcp, these syscall handlers are very light:

#[no_mangle]
pub extern "C" fn rust_net_listen(port: i32) -> i32 {
    unsafe {
        let sockets = match SOCKET_SET.as_mut() {
            Some(s) => s,
            None => return -1,
        };
        let handle = match TCP_HANDLE {
            Some(h) => h,
            None => return -1,
        };
        let socket = sockets.get_mut::<tcp::Socket>(handle);
        match socket.listen(port as u16) {
            Ok(()) => 0,
            Err(_) => -1,
        }
    }
}

#[no_mangle]
pub extern "C" fn rust_net_accept() -> i32 {
    unsafe {
        let sockets = match SOCKET_SET.as_mut() {
            Some(s) => s,
            None => return -1,
        };
        let handle = match TCP_HANDLE {
            Some(h) => h,
            None => return -1,
        };
        let socket = sockets.get_mut::<tcp::Socket>(handle);
        use smoltcp::socket::tcp::State;
        match socket.state() {
            State::Established => 1,
            State::Listen | State::SynReceived | State::SynSent => 0,
            _ => 0,
        }
    }
}

#[no_mangle]
pub extern "C" fn rust_net_recv(buf: *mut u8, len: i32) -> i32 {
    if buf.is_null() || len <= 0 {
        return -1;
    }
    unsafe {
        let sockets = match SOCKET_SET.as_mut() {
            Some(s) => s,
            None => return -1,
        };
        let handle = match TCP_HANDLE {
            Some(h) => h,
            None => return -1,
        };
        let socket = sockets.get_mut::<tcp::Socket>(handle);

        if !socket.may_recv() {
            return 0;
        }

        let max_len = len as usize;
        let mut out_n: i32 = 0;

        let res = socket.recv(|data| {
            let n = core::cmp::min(data.len(), max_len);
            if n > 0 {
                core::ptr::copy_nonoverlapping(data.as_ptr(), buf, n);
            }
            out_n = n as i32;
            // consume n bytes, return n as the closure result
            (n, ())
        });

        if res.is_err() {
            -1
        } else {
            out_n
        }
    }
}

#[no_mangle]
pub extern "C" fn rust_net_send(buf: *const u8, len: i32) -> i32 {
    if buf.is_null() || len <= 0 {
        return -1;
    }
    unsafe {
        let sockets = match SOCKET_SET.as_mut() {
            Some(s) => s,
            None => return -1,
        };
        let handle = match TCP_HANDLE {
            Some(h) => h,
            None => return -1,
        };
        let socket = sockets.get_mut::<tcp::Socket>(handle);

        // Only send if TCP state/window allows
        if !socket.may_send() {
            return 0; // would block / not ready
        }

        let slice = core::slice::from_raw_parts(buf, len as usize);

        match socket.send_slice(slice) {
            Ok(n) => n as i32, // actual bytes queued
            Err(_) => -1,
        }
    }
}


#[no_mangle]
pub extern "C" fn rust_net_close() {
    unsafe {
        let sockets = match SOCKET_SET.as_mut() {
            Some(s) => s,
            None => return,
        };
        let handle = match TCP_HANDLE {
            Some(h) => h,
            None => return,
        };
        let socket = sockets.get_mut::<tcp::Socket>(handle);
        socket.close();
    }
}

Testing & Conclusion

From my linux host, I’ve then created a tap device:

sudo modprobe tun
sudo ip tuntap add dev tap0 mode tap user "$USER"
sudo ip addr add 10.0.3.1/24 dev tap0

And put in on the 10.0.3.1/24 network. On the xv6 device I’ve then hardcoded an ip and MAC address (I don’t have a dhcp server running on my host):

const MY_MAC_BYTES: [u8; 6] = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee];
const MY_IP_V4: Ipv4Address = Ipv4Address::new(10, 0, 3, 2);

And asked ChatGPT to write a simple HTTP server using the syscalls written above:

//server.c, generated by ChatGPT 5.1
#include "types.h"
#include "stat.h"
#include "user.h"

// Returns 1 if request body == "close"
static int
body_is_close(char *buf, int n)
{
  if(n <= 0) return 0;

  // find "\r\n\r\n" manually
  int i;
  for(i = 0; i+3 < n; i++){
    if(buf[i]   == '\r' &&
       buf[i+1] == '\n' &&
       buf[i+2] == '\r' &&
       buf[i+3] == '\n')
    {
      // body begins after the blank line
      int body_start = i + 4;

      // skip spaces/newlines
      while(body_start < n &&
           (buf[body_start] == ' ' ||
            buf[body_start] == '\n' ||
            buf[body_start] == '\r'))
        body_start++;

      // check for "close"
      if(body_start + 5 <= n &&
         buf[body_start]   == 'c' &&
         buf[body_start+1] == 'l' &&
         buf[body_start+2] == 'o' &&
         buf[body_start+3] == 's' &&
         buf[body_start+4] == 'e')
        return 1;

      return 0; // found body, but not "close"
    }
  }
  return 0; // didn't find end of headers
}

int
main(void)
{
  for(;;){
    // Put the TCP socket back into LISTEN state each iteration.
    // On first iteration the socket is Closed; later it is Closed again
    // after net_close().
    if(net_listen(80) < 0){
      printf(2, "httpd: net_listen(80) failed\n");
      exit();
    }

    printf(1, "httpd: listening on port 80\n");

    // Wait until the TCP connection reaches Established
    while(net_accept() == 0)
      sleep(10);

    printf(1, "httpd: connection established\n");

    char buf[512];
    int n = net_recv(buf, sizeof(buf));

    if(n > 0)
      printf(1, "httpd: received %d bytes\n", n);

    if(body_is_close(buf, n)){
      // Shutdown reply
      static const char hdr[] =
        "HTTP/1.0 200 OK\r\n"
        "Content-Length: 13\r\n"
        "Content-Type: text/plain\r\n"
        "\r\n";
      static const char body[] = "shutting down\n"; // 13 bytes

      net_send((void*)hdr,  sizeof(hdr)  - 1);
      net_send((void*)body, sizeof(body) - 1);
      net_close();

      printf(1, "httpd: close command received, exiting.\n");
      exit();
    } else {
      // Normal reply
      static const char hdr[] =
        "HTTP/1.0 200 OK\r\n"
        "Content-Length: 13\r\n"
        "Content-Type: text/plain\r\n"
        "\r\n";
      static const char body[] = "hi from user\n"; // 13 bytes

      net_send((void*)hdr,  sizeof(hdr)  - 1);
      net_send((void*)body, sizeof(body) - 1);
      net_close();

    }
  }
}

This server returns a simple message when a GET request is sent, and shuts down when someones POSTs “close”.

On the left: qemu running xv6 with rust networking, on the right: linux host connecting to the server. The final ls output shows that sending a close in the post payload shuts down the server.

Code & Patch

The code is accessible at github.com/Ferryistaken/xv6-public, and a full patch is pasted below.

⤓ patch.diff

Thanks & Acknowledgements

Special thanks to my CS3210: Design of Operating Systems professor Michael Specter, this OS guide, and the smoltcp dev team.

In this short guide I will go over adding Rust “modules” to a base xv6 kernel for the x86 target (although it could be adapted to RISC-V with minimal changes).

While xv6 doesn’t really support “modules” in the same way that the Linux kernel does, here I describe a module simply as a subsystem of the kernel that in this case will be completely done in Rust.

Why?

With this approach you gain memory safety in all the code you abstract to Rust, and only stay unsafe in:

1. The handoff of data from C to Rust and back.
2. MMIO and DMA whenever it’s used.

Gaining:

1. No out-of-bounds memory access.
2. No dangling pointers.
3. No double-free.

Setup

This approach uses:

• A standalone Rust crate (rustmod/)
• A static library (librustmod.a1)
• CMake glue to build Rust before linking the kernel
• A minimal C → Rust → C FFI interface

Rust crate

Inside of the root xv6 project, create a rustmod crate:

mkdir rustmod
cd rustmod
cargo init --lib

Cargo

And make this your Cargo.toml:

[package]
name = "rustmod"
version = "0.1.0"
edition = "2021"

[lib]
crate-type = ["staticlib"]

[profile.release]
panic = "abort"
lto = true
codegen-units = 1

And make this your .cargo/config.toml

[build]
target = "i686-unknown-linux-gnu"

[target.i686-unknown-linux-gnu]
linker = "gcc"
rustflags = [
  "-C", "relocation-model=static",
  "-C", "link-arg=-m32",
  "-C", "target-feature=-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2",
  "-C", "soft-float",
]

This will make cargo to emit a .a file suitable for linking with the xv6 kernel.

Rust

Let’s write a basic Rust function to test that everything is working correctly:

rustmod/src/lib.rs

#![no_std]

use core::panic::PanicInfo;

extern "C" {
    fn c_kputs(msg: *const u8);
}

#[no_mangle]
pub extern "C" fn rust_test() {
    static MSG: &[u8] = b"Rust support loaded\n\0";

    unsafe {
        c_kputs(MSG.as_ptr());
    }
}

#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
    loop {}
}

• #![no_std] is required because the kernel provides no standard library.
• #[no_mangle] and extern "C" ensure ABI compatibility with xv6.
• panic = "abort" avoids Rust’s complex unwinding.

C Glue

Inside kernel/include/defs.h add:

// external rust
extern void rust_test(void);

And in console.c add:

void
c_kputs(const char *s)
{
  cprintf("%s", s);
}

Finally, in main() add the rust_test(); function:

int
main(void)
{
  ...
  userinit();
  rust_test();
  mpmain();
}

Building & Linking

The xv6 kernel is built using kernel/CMakeLists.txt. This file has to be modified such that it:

• Builds the Rust crate via a custom target.
• Links the resulting static library into the kernel ELF.
• Adds correct dependencies so Rust always builds before the kernel.

Building Rust target in CMake

Add this to kernel/CMakeLists.txt

# Path to the rust staticlib
set(RUSTMOD_DIR ${CMAKE_SOURCE_DIR}/rustmod)
set(RUSTMOD_TARGET i686-unknown-linux-gnu)
set(RUSTMOD_LIB ${RUSTMOD_DIR}/target/${RUSTMOD_TARGET}/release/librustmod.a)

# Build the Rust kernel module with cargo
add_custom_target(rustmod_build
    COMMAND cargo build --release
    WORKING_DIRECTORY ${RUSTMOD_DIR}
    BYPRODUCTS ${RUSTMOD_LIB}
    COMMENT "Building Rust kernel module"
)

Which defines:

cargo build --release --target i686-unknown-linux-gnu

To output librustmod.a.

Linking into xv6 kernel

Find wherever you’re linking the kernel and all kernel objects:

COMMAND ld -m elf_i386 -nostdlib -T kernel.ld ...
        ${kernel_OBJECTS} -b binary initcode entryother

And change it with:

COMMAND ld -m elf_i386 -nostdlib -T ${CMAKE_CURRENT_SOURCE_DIR}/kernel.ld -o kernel
        ${kernel_OBJECTS} ${RUSTMOD_LIB} -b binary initcode entryother
DEPENDS
        ...
        rustmod_build

Testing Functionality

Finally, when building and starting xv6, you should see:

SeaBIOS (version 1.15.0-1)


iPXE (https://ipxe.org) 00:03.0 CA00 PCI2.10 PnP PMM+1FF8B4A0+1FECB4A0 CA00



Booting from Hard Disk..xv6...
Rust support loaded.             <-- From Rust!
cpu0: starting 0
sb: size 20000 nblocks 19937 ninodes 200 nlog 30 logstart 2 inodestart 32 bmap start 58
init: starting sh
$ ls
.              1 1 512
..             1 1 512
init           2 2 46876
README         2 3 2170
sh             2 4 54604
ls             2 5 48684
cat            2 6 47016
rm             2 7 46448
test_sha256    2 8 48236
test_aes256    2 9 47872
console        3 10 0
$

First off, make sure to remove previously install displaylink and evdi packages.

sudo dnf remove displaylink 'evdi-*'

And install tools to build packages from source.

sudo dnf groupinstall 'Development Tools'

Building evdi

Now build and install the evdi modules.

git clone https://github.com/DisplayLink/evdi
cd evdi
export CPLUS_INCLUDE_PATH="/usr/include/python3.12:$CPLUS_INCLUDE_PATH"
make
sudo make install

Installing displaylink

Now, download the right displaylink RPM version from the displaylink github.

Both the built and src rpm should work, but this guide is going to use the x86_64 version.

sudo dnf install '/path/to/rpm/fedora-40-displaylink-1.14.4-1.github_evdi.x86_64.rpm

Finally reboot. Now displaylink should work. This guide works for Fedora 38, 39, and possibly any new version, given that you pull the latest evdi version, and download the correct displaylink driver rpm.

Inspired by this post, I chose to create my own solution to automatically export my book highlights on a website and map them onto a vector space in order to have features such as semantic search.

The process is all serverless, free, and automated. As a bonus, I also get 3 highlights sent to my inbox every day. And again, my total cost for this whole infrastructure is $0.

Here’s the final result: books.alessandroferrari.live

And a live representation of my highlights:

The whole system is structured in such a way that every time the google sheet gets updated, the website reflects it after just a few minutes. The system is open an expandable, and features such as emailing yourself your own quotes can be implemented trivially.

Exporting book highlights onto a website

There are plenty of resources that explain how to automatically export highlights taken on platforms such as kindle or iBook as a csv format, so I won’t cover that step of the process. It only takes 5 minutes per book to do it manually anyways, and the process varies drastically depending on which reading app you use, for this reason I haven’t included any book exporting pipeline in my system.

The important part is: all your highlights should be held in a google sheet formatted like this:

All I keep is the actual highlight and the isbn. Anything else is just useless data to move around. I have been thinking about keeping highlight notes in this csv as well, however I like the idea of the website being pure quotes, with my own thoughts on those quotes being stored somewhere else.

The next step is to somehow put these quotes onto a website. Going from this google sheet to a published website is a 4 step process:

Steps

1. Getting the data as a csv file
2. Break up and format the data in a way that is understandable by jekyll
3. Generate embeddings from this data
4. Deploy the generated site on a hosting service

Getting the Data

I found out recently that you can just curl google sheets documents as csv files with this command:

curl -L <google sheet url> -o sheet.csv

This completes part 1. The only caveat is that the google sheets needs to be readable by anyone with the link, which isn’t something that I particularly care about as everything on that sheet will already be published onto the website.

Formatting the data

Since I already use Jekyll for the main section of my website I chose to use jekyll for this section too. It’s simple and I’m somewhat familiar with it, however this same process could be replicated with any SSG (or even a webdev framework).

I have a simple python script that generates the jekyll pages based on the url of the spreadsheet. It also adds the authors, titles, cover images, and publishing dates based on google books API. You can find the script here.

For the script to work, make sure that you google books API key is in the environment variable GOOGLE_API_KEY.

The script will generate markdown files of this format:

---
layout: post
title: "The Sailor Who Fell from Grace with the Sea"
authors: "Yukio Mishima"
publisher: "Vintage"
publishedDate: "1994-05-31"
coverImage: "http://books.google.com/books/content?id=B-eBAAAAIAAJ&printsec=frontcover&img=1&zoom=1&source=gbs_api"
highlights:
  - "While Noboru wandered through private dreams Tsukazaki stood at Fusako’s side, and the heat of his body in the sultry chart room was beginning to oppress her: when the parasol she had leaned against a desk clattered suddenly to the floor, she felt as if she herself, fainting, had fallen."
  - "He hadn’t been able to explain his ideas of glory and death, or the longing and the melancholy pent up in his chest, or the other dark passions choking in the ocean’s swell."
  - "He ..."
---

And this is the post layout that this script uses:

---
layout: default
---

<!DOCTYPE html>
<html>
<head>
    <title>Embedding book highlights onto a vector space</title>
    <script>
    function copyToClipboard(elementId, element) {
        var url = window.location.href.split('#')[0] + '#' + elementId;
        navigator.clipboard.writeText(url).then(function() {
            element.classList.add("clicked");

            setTimeout( function() {
                element.classList.remove("clicked");
            }, 100);
        }).catch(function(err) {
            console.error('Could not copy text: ', err);
        });
    }
    </script>
</head>
<body>
    <div style='text-align: center;'>
        <img src='' alt='Embedding book highlights onto a vector space' style='max-width: 80%;'>
    </div>

    <h2 style='text-align: center; font-weight: bold; font-size: 24px;'>Embedding book highlights onto a vector space</h2>
    <p style='text-align: center;'><br></p>

    <h3 style='text-align: center;'>Highlights</h3>
    <ul style="list-style-type: none; text-align: center; padding: 0;">
        
    </ul>

    <a href="/">Back to Home</a>
</body>
</html>

Already, this creates a perfectly functioning website in which all of your book highlights will be displayed. I wanted to go a step further:

Generating embeddings from the book highlights

Now to part 3: generating embeddings for features such as clustering and semantic search.

I have a second script, embeddings.py which takes in the downloaded google sheet and generates embeddings using sentence-transformers. The best performing model that I have found is all-MiniLM-L6-v2, a model that maps sentences and paragraphs to a 384 dimensional dense vector space.

This is the script to generate embeddings:

print("Loading libraries")

import pandas as pd
from sentence_transformers import SentenceTransformer
import json
import umap.umap_ as umap
import matplotlib.pyplot as plt
import numpy as np
import requests
import os

def get_book_title(isbn):
    api_key = os.environ.get('GOOGLE_API_KEY')
    url = f"https://www.googleapis.com/books/v1/volumes?q=isbn:{isbn}&key={api_key}"
    response = requests.get(url)

    if response.status_code != 200:
        return "Title Not Found"

    data = response.json()
    if "items" in data:
        try:
            title = data["items"][0]["volumeInfo"]["title"]
            return title
        except (IndexError, KeyError):
            return "Title Not Found"
    else:
        return "Title Not Found"

print("Loading Model")

# Load the model
model = SentenceTransformer("all-MiniLM-L6-v2")
model.max_seq_length = 256

print("Loading data")

# Read the CSV file
df = pd.read_csv('sheet.csv')

# Maintain the original 'index' column
df['original_index'] = df.index

# Group by ISBN and create indices within each group
df['group_index'] = df.groupby('isbn').cumcount()

# Extract sentences (highlights/quotes), ISBNs, and group indices
sentences = df['highlight'].tolist()
isbns = df['isbn'].tolist()
group_indices = df['group_index'].tolist()

print("Encoding data")

# Generate embeddings
embeddings = model.encode(sentences, normalize_embeddings=True)

# Convert embeddings to list for JSON compatibility and save to JSON
embeddings_list = embeddings.tolist()
data = {
    "sentences": sentences, 
    "embeddings": embeddings_list, 
    "isbns": isbns, 
    "group_indices": group_indices
}

print("Saving embeddings")

with open("embeddings.json", "w") as file:
    json.dump(data, file)

print("Generating visualization")

# Dimensionality reduction using UMAP
umap_embeddings = umap.UMAP(n_neighbors=15, n_components=2, metric='cosine').fit_transform(embeddings)

markers = ['o', 's', '^', 'D', '*', 'x', '+', '>', '<', 'p', 'h', 'H', 'X', 'd']

# Creating a color map and fetching titles for each unique ISBN
unique_isbns = list(set(isbns))
isbn_to_title = {isbn: get_book_title(isbn) for isbn in unique_isbns}
colors = plt.cm.jet(np.linspace(0, 1, len(unique_isbns)))
isbn_to_color = {isbn: color for isbn, color in zip(unique_isbns, colors)}

plt.style.use('https://github.com/dhaitz/matplotlib-stylesheets/raw/master/pitayasmoothie-dark.mplstyle')

# Plotting
plt.figure(figsize=(12,10))
legend_elements = []

for idx, isbn in enumerate(unique_isbns):
    indices = [i for i, x in enumerate(isbns) if x == isbn]
    marker_style = markers[idx % len(markers)]  # Cycle through markers

    # Scatter plot
    scatter = plt.scatter(umap_embeddings[indices, 0], umap_embeddings[indices, 1], s=30, color=isbn_to_color[isbn], marker=marker_style)

    # Calculating centroid of each cluster
    centroid_x = np.mean(umap_embeddings[indices, 0])
    centroid_y = np.mean(umap_embeddings[indices, 1])

    # Annotate with book title at the centroid
    plt.annotate(isbn_to_title[isbn], (centroid_x, centroid_y), fontsize=9, ha='center', va='center', color='white')

    # Create a custom legend entry for each ISBN
    legend_elements.append(plt.Line2D([0], [0], marker=marker_style, color='w', label=isbn_to_title[isbn], markersize=10, markerfacecolor=scatter.get_facecolor()[0], linestyle='None'))

plt.title('Vector Space (UMAP)', fontsize=20)

# Add a legend and adjust its properties
legend = plt.legend(handles=legend_elements, title='', loc='upper right', bbox_to_anchor=(1, 0.9), bbox_transform=plt.gcf().transFigure)
legend.get_frame().set_alpha(0.5)  # Adjust the opacity

plt.grid(False)
plt.axis('off')

# Save the plot as a PNG file
plt.savefig("umap.png", dpi=300, bbox_inches='tight')

This script does two things:

1. Downloads the highlights and generates embeddings from it.
2. Visualizes this embeddings through UMAP.

This is enough to visualize all the highlights and cluster the books. As can be seen from the UMAP visualization, highlights from the same book cluster together and books on the same topic also cluster together, which is very interesting to see.

Semantic Search

The second part of this system is the script that takes user inputs and queries the vector space for similar highlights. This is done through the following serverless netilfy function:

const math = require('mathjs');

let pipeline;

(async () => {
  const transformers = await import('@xenova/transformers');
  pipeline = transformers.pipeline;
})();

exports.handler = async (event) => {
    // URL of the embeddings JSON file
    const embeddingsUrl = 'https://books.alessandroferrari.live/embeddings.json';

    // Fetch the embeddings data from the URL
    const response = await fetch(embeddingsUrl);
    const data = await response.json();

    const sentences = data.sentences;
    const embeddings = data.embeddings;
    const isbns = data.isbns; // Include the ISBNs
    const indexes = data.group_indices;

    console.log(isbns);
    console.log(indexes);

    // Extract query from event
    const query = event.queryStringParameters.q;

    // Initialize the feature extraction pipeline
    let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');

    // Calculate query embedding
    const queryEmbeddingOutput = await extractor(query, { pooling: 'mean', normalize: true });
    const queryEmbedding = queryEmbeddingOutput[0]; // Extract the embedding

    const arrayQueryEmbedding = Array.from(queryEmbedding.data);

    console.log(arrayQueryEmbedding);
    //console.log(embeddings.every(embedding => Array.isArray(embedding))); // Should also be true


    // Calculate similarities
    let scores = embeddings.map(embedding => {
        return math.dot(arrayQueryEmbedding, embedding) / 
               (math.norm(arrayQueryEmbedding) * math.norm(embedding));
    });

    // Find top results
    let indices = scores.map((score, index) => [score, index]);
    indices.sort((a, b) => b[0] - a[0]);
    let topResults = indices.slice(0, 5);

    console.log(topResults);

    // Prepare response
    const results = topResults.map(item => {
        return {
            sentence: sentences[item[1]],
            similarity: item[0],  // Add the similarity score
            isbn: isbns[item[1]], // Include the ISBN
            index: indexes[item[1]]
        };
    });

    return {
        statusCode: 200,
        body: JSON.stringify(results)
    };

};

It’s important to use the same embedding model, otherwise the query will be mapped differently and won’t be of much use.

Through this function I implemented semantic search, meaning that I can search for things such as “Stoicism in financial markets” or “Finding meaning through hard work” and these will be the responses:

Interactively visualizing vector embeddings

I however wanted to go a step further: I wanted to see a live representation of my highlights, so I wrote a simple python script that lets me see the vector space of the highlights in an interactive manner. Here is live-embeddings.py:

import json
import plotly.graph_objects as go
import plotly.express as px
import pandas as pd
import umap.umap_ as umap
import numpy as np
import os
import requests
import textwrap

# Function to fetch book titles using Google Books API
def get_book_title(isbn):
    api_key = os.environ.get('GOOGLE_API_KEY')
    url = f"https://www.googleapis.com/books/v1/volumes?q=isbn:{isbn}&key={api_key}"
    response = requests.get(url)

    if response.status_code != 200:
        return "Title Not Found"

    data = response.json()
    if "items" in data:
        try:
            title = data["items"][0]["volumeInfo"]["title"]
            return title
        except (IndexError, KeyError):
            return "Title Not Found"
    else:
        return "Title Not Found"

# Load data from embeddings.json
with open("embeddings.json", "r") as file:
    data = json.load(file)

embeddings = np.array(data["embeddings"])
sentences = data["sentences"]
isbns = data["isbns"]

# Dimensionality reduction using UMAP
umap_embeddings = umap.UMAP(n_neighbors=15, n_components=2, metric='cosine').fit_transform(embeddings)

# Get unique ISBNs and their corresponding titles
unique_isbns = list(set(isbns))
isbn_to_title = {isbn: get_book_title(isbn) for isbn in unique_isbns}
colors = px.colors.qualitative.Plotly

highlight_ids = {isbn: 0 for isbn in unique_isbns}

# Function to generate URL and update highlight IDs
def generate_url(isbn):
    url = f"https://books.alessandroferrari.live/{isbn}#{highlight_ids[isbn]}"
    highlight_ids[isbn] += 1
    return url

# Generate URLs for each highlight
urls = [generate_url(isbn) for isbn in isbns]

# Wrap text for each highlight and append URL
wrapped_sentences_with_url = [
    '<br>'.join(textwrap.wrap(sentence, width=80)) + f"<br>URL: {url}" 
    for sentence, url in zip(sentences, urls)
]

# Prepare Plotly data
plot_data = pd.DataFrame({
    'x': umap_embeddings[:, 0],
    'y': umap_embeddings[:, 1],
    'text': wrapped_sentences_with_url,
    'isbn': isbns
})

# Create Plotly figure
fig = go.Figure()

for idx, isbn in enumerate(unique_isbns):
    isbn_data = plot_data[plot_data['isbn'] == isbn]
    fig.add_trace(go.Scatter(
        x=isbn_data['x'],
        y=isbn_data['y'],
        mode='markers',
        marker=dict(color=colors[idx % len(colors)], size=10),
        text=isbn_data['text'],  # Include the wrapped text with URL
        hoverinfo='text',       # Only show the text on hover
        name=isbn_to_title[isbn]
    ))

# Update layout
fig.update_layout(
    title='',
    plot_bgcolor='white',
    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)
)

# Save the figure as an HTML file
fig.write_html("plotly-out.html")

This simple script generates an html file that can be seen here. From this I can see all my highlights by hovering my mouse over the dots.

As a cherry on top, a netlify build can be triggered every time the google sheet gets updated through a simple Google AppScript and AppScript trigger:

function triggerNetlifyBuild() {
var scriptProperties = PropertiesService.getScriptProperties();
var lastDeployTime = scriptProperties.getProperty('lastDeployTime');
var currentTime = new Date().getTime();

// Check if last deploy was less than 5 minutes ago
if (lastDeployTime !== null && (currentTime - lastDeployTime) < 5 * 60 * 1000) {
	Logger.log('Deploy skipped to prevent spamming. Less than 5 minutes since last deploy.');
	return;
}

scriptProperties.setProperty('lastDeployTime', currentTime.toString());

// Your Netlify hook URL
var netlifyHookUrl = scriptProperties.getProperty('NETLIFY_BUILD_ID')

UrlFetchApp.fetch(netlifyHookUrl, { method: 'POST' });
Logger.log('Netlify build triggered.');
}

This script sends a post request to a netlify build hook, and can be triggered every time the script gets edited. I also added a countdown to avoid spam.

Email quotes to your inbox

Since this whole system is very expandable and modular, I implemented another cool feature that I saw in products such as Readwise: getting your own quotes in your inbox for recollection and resurfacing old concepts. I implemented this using Google AppScript, making it again free and serverless.

I have a trigger that runs every day between 6am and 7am which picks 3 random quotes from the sheet and formats them nicely in my inbox. Here are the functions:

function getBookInfoByISBN(isbn) {
	var scriptProperties = PropertiesService.getScriptProperties();
	var API_KEY = scriptProperties.getProperty('BOOK_API');
	var url = 'https://www.googleapis.com/books/v1/volumes?q=isbn:' + isbn + '&key=' + API_KEY + '&country=US';
	var response = UrlFetchApp.fetch(url);
	var json = JSON.parse(response.getContentText());
	
	if (json.totalItems > 0) {
		var book = json.items[0].volumeInfo;
		return { title: book.title, author: book.authors ? book.authors.join(", ") : "Unknown Author" };
	} else {
		return { title: 'Unknown Title', author: 'Unknown Author' };
	}
}

function sendRandomQuotesEmail() {
	var sheet = SpreadsheetApp.getActiveSpreadsheet().getActiveSheet();
	var data = sheet.getDataRange().getValues();
	var randomQuotes = getRandomQuotes(data, 3);
	
	var message = '<div style="font-family: Arial, sans-serif; font-size: 14px;">';
	
	randomQuotes.forEach(function(quote) {
		// Each quote with highlighted background and citation in standard text
		message += `<p style="word-wrap: break-word;">` +
		`<span style="background-color: rgba(255, 226, 130, 0.5);">${quote.text}</span>` +
		`</p>` +
		`<p style="font-style: italic; margin-top: 5px;">—${quote.title} by ${quote.author}</p>`;
	});
	
	// End of the message
	message += '</div>';
	
	MailApp.sendEmail({
		to: "<your email>",
		subject: "Your Daily Book Quotes",
		htmlBody: message
	});
}

function getRandomQuotes(data, numberOfQuotes) {
	var quotes = [];
	var usedIndices = [];
	
	while (quotes.length < numberOfQuotes) {
		var randomIndex = Math.floor(Math.random() * data.length);
		if (usedIndices.indexOf(randomIndex) === -1) {
			usedIndices.push(randomIndex);
			var isbn = data[randomIndex][1];
			var bookInfo = getBookInfoByISBN(isbn);
			var quoteText = data[randomIndex][0];
			
			quotes.push({
				text: quoteText,
				title: bookInfo.title,
				author: bookInfo.author
			});
		}
	}
	return quotes;
}

This way I get a daily email with 3 highlights that looks like this:

Summary

Wrapping up, this project is a cost-effective solution for transforming book highlights into an interactive, searchable vector space on my website. The system, entirely serverless and automated, updates in real-time with each addition to a Google Sheet. It also leaves space for plugins and extensions, such as the three daily highlights that land in my inbox.

The core of this project lies in how these highlights are not just static text but dynamic data points in a vector space, enabling semantic search. This integration of technology with literature not only highthens the reading experience but also demonstrates the practical application of machine learning in personal projects.

Although this project is public on the internet, I will probably be the one benefitting from it the most, as I like looking back at my own highlights.

The project is currently live at books.alessandroferrari.live. And the source code for it can be found on github.

The thought of creating a personal brand, building an audience, and curating content that is relevant to what I care about and research is more tempting than ever.

I think that there is a case for me to take time out of my days to share my views and expertise on different subjects that I’m well versed in, as I might genuinely create value for some of my readers (I’ll define what I mean by create value later).

However I view time as my scarcest (and most valuable) commodity, and taking time out of my research, studies, side projects, job search, etc, etc, etc in order to curate my thoughts is often hard to justify in my own mind.

Either choice seems a no-brainer. I should start curating content because:

• People that I respect and are extremely successful in their own fields do it (Huberman, Walker, Housel, etc).
• I already write on my own, almost every day, so it seems natural to just post what I write.
• It’s good to have a personal brand. Ideally when someone googles your name you’re the first person that pops up, otherwise you’re being dominated in the late-stage capitalism hierarchy :).
• I like the idea of having an audience that I create value for.

However, it’s a bad idea to start curating content because:

• It takes time to create value. It takes a long time to even get to the point in your life where you’re in a position to create value, and then it takes more time to create value once you’re in that position.
• What if you’re falling in the The Creative World’s Bullshit Industrial Complex trap?
• It’s scary. Does your voice really need to be heard? What if you write something that you don’t agree with in 3 years? What about in 10 years?
• The positive outcomes aren’t clear. Are you getting payed? Are you getting recognition? Will this help you academically, professionally, spiritually or in any other way?

While thinking about this, I came up with an idea:

The Sunk Cost Opportunity

We are all familiar with the sunk cost fallacy: our tendency to follow through on something if we are already invested in it. To this fun concept, I raise: the sunk cost opportunity.

On any given week, I spend tens of hours researching random topics. Some weeks I read publications on the effect on noise machines on sleep, some other weeks I’ll sink time into building a “smart” business card for no apparent benefit, at other times I will code an emulator to learn Rust.

These are all endeavours that feed my constant research for knowledge. They are almost therapeutic. They keep me sharp. They keep me updated on current events. They even train my ability to pivot from one project to another quickly–a nice switch up from the semester or year-long projects that I’m faced with in my school or research.

Point is: I am already at peace with committing 7 hours of my week researching the effects of white noise on my sleep. I do it for fun. Those hours are a cost that I’m willing to sink at any time of my life–there is no friction at all in spending them however I see fit.

Naturally, after this realization, my next thought was: why don’t I spend another 20 minutes on top of those 7 hours curating a simple blog post, a thread on twitter, or even simply a well-written journal entry that no one but me will read?

This is the essence of the sunk cost opportunity: how can I create the most amount of value from cost that is already “sunk” in order to generate an asymmetrically large outcome.

how can I create the most amount of value from cost that is already “sunk” in order to generate an asymmetrically large outcome.

I’m not going to spend 4 hours researching something just to publish it to my audience. I’m not a writer, philosopher, or influencer, however I’m perfectly ok with spending 30 minutes writing about something that I already commit hours to for others reasons. That’s perfectly reasonable in my head.

Creating Value

This way I’m also sure to create value. There surely is someone else like me, who has the same thirst for knowledge in similar topics, who will enjoy not having to spend 7 hours researching the effect of white noise on sleep. Someone who would rather read my blog post about it and invest his weekly personal-research hours into something else.

On the other side of things, there will also be someone that reads what I have to say and chooses to go on an even deeper dive, however that person will now have a good starting point.

These are the archetypes of people that I’m creating value for: the student in my same position who will now be able to devote his hours to something else. The kid that is trying to get in the same intersection of CS and Biology as me and that won’t need to go through as many hurdles to get to where I’m at. Or the person that reads my content and chooses that they want to delve deeper into the subject.

I find content valuable if:

• It saves me hours of time.
• It makes me waste hours of time because I’m so captivated by it that I need more of it.

And my content is aimed at provoking the same outcome in other people.

I find content valuable if it saves me hours of time or of it makes me waste hours of time researching more about. No in-between.

PS: This is a great read that in part motivated me to write this. His whole blog is great.

⭐️ Build a business, not an audience

When doing single cell multiome analysis (ATAC + RNA) in R using packages like Seurat or ArchR, it is useful to load both the scRNA-seq and scATAC-seq datasets in the same object under the same framework.

This however brings some problems, as ArchR doesn’t have built-in functions or tutorials on how to filter out bad RNA reads from the ArchR object, and vice versa from the Seurat object.

This is an easy problem to fix, however I haven’t found any easy resources online that explain the very simple process, and chose to write my own.

Loading the data

For this tutorial, I’m going to use ArchR as the main framework. I first load in the data like this:

inputFiles <- "data/<your-file-name>"

ArrowFiles <- createArrowFiles(
  inputFiles = inputFiles,
  minTSS = 0,
  minFrags = 500, 
  addTileMat = TRUE,
  addGeneScoreMat = TRUE
)

proj <- ArchRProject(ArrowFiles)

seRNA <- import10xFeatureMatrix(
  input = c("data/pbmc_unsorted_10k_filtered_feature_bc_matrix.h5"),
  names = c("PBMC_10k")
)

proj <- addGeneExpressionMatrix(input = proj, seRNA = seRNA, force = TRUE)

# Filter out NA Reads
proj <- proj[!is.na(proj$Gex_nGenes) & 
               !is.na(proj$Gex_MitoRatio)]

Quality Control

Then, I start my quality control pipeline. In this case, I want to set the minTSS to 12, minFrags to 1200, minFeatures to 600, maxFeatures to 4000 and maxMT to 0.019.

minTSS <- 12
minFrags <- 1200

minFeatures <- 600
maxFeatures <- 4000
maxMT <- 0.019

ATAC Quality Control

In order to filter out ATAC cells that don’t pass these thresholds, I can index into the ArchR object like so:

proj <- proj[proj$TSSEnrichment > minTSS & proj$nFrags > minFrags]

RNA Quality Control

I can then repeat the same process for the RNA data.

proj <- proj[proj$Gex_nGenes > minFeatures
             & proj$Gex_nGenes < maxFeatures
             & proj$Gex_MitoRatio < maxMT]

Conclusions

This might be obvious for a lot of people, and it is after getting some experience with bioinformatics. I just wanted to make this short post to make the knowledge more accessible, as I don’t think that anyone should lose 30 minutes trying to figure this out, like I did.

Introduction

This post isn’t supposed to be a step-to-step guide, it may be treated as such, but you will probably find some better guides on the internet. I wrote this emulator a while ago and am just now(~1.5 years later) writing this post, so don’t lynch me for my errors–or better yet; fix them on github.

Skip to the code.

Project setup

CHIP8

CHIP8 Description

16 8-bit registers

Registers are small storage locations in the CPU for temporary data storage and operations.

let mut registers: [u8; 16] = [0; 16]; // Registers V0 to VF

4k bytes of memory

Memory is the larger storage used for program instructions, long-term data, and short-term data.

let mut memory: [u8; 4096] = [0; 4096];

16-bit index registers

The Index Register holds memory addresses for certain operations.

let mut index_register: u16 = 0;

16-bit program counter

The Program Counter keeps track of the address for the next instruction to execute.

let mut program_counter: u16 = 0x200; // The program starts at 0x200

16-level stack

The stack keeps track of return addresses when calling into functions.

let mut stack: [u16; 16] = [0; 16];

8-bit stack pointer

The Stack Pointer points to the top-most level of the stack.

let mut stack_pointer: u8 = 0;

8-bit delay timer

The delay timer in CHIP-8 decrements at a rate of 60Hz until it reaches zero.

// 8-bit delay timer
let mut delay_timer: u8 = 0;

8-bit sound timer

When the sound timer is set to a value greater than zero, a beep is produced and it decrements at a rate of 60Hz until it reaches zero.

// 8-bit sound timer
let mut sound_timer: u8 = 0;

16 input keys

CHIP-8 has 16 input keys, typically represented by the hexadecimal numbers 0 through F.

// Keys 0x0 through 0xF.
let mut keypad: [u8; 16] = [0; 16];

64x32 monochrome display memory

CHIP-8 has a monochrome display of 64x32 pixels.

// Display, 64x32.
let mut video: [u32; 64 * 32] = [0; 64 * 32];

Implementation Specific Variables

Opcode Function Tables

For a CHIP-8 emulator, using function tables is a common and efficient way to handle opcodes. By mapping each opcode to a specific function that implements it, you can avoid a large switch or match statement.

// Main opcode table
let mut table: [fn(&mut Chip8); 0xF+1] = [Chip8::OP_ERR; 0xF+1];

// Sub-tables for opcodes that begin with 0, 8, E, and F
let mut table0: [fn(&mut Chip8); 0xE+1] = [Chip8::OP_ERR; 0xE+1];
let mut table8: [fn(&mut Chip8); 0xE+1] = [Chip8::OP_ERR; 0xE+1];
let mut tableE: [fn(&mut Chip8); 0xE+1] = [Chip8::OP_ERR; 0xE+1];
let mut tableF: [fn(&mut Chip8); 0x65+1] = [Chip8::OP_ERR; 0x65+1];

• table: This is the primary opcode table. Each index of this array corresponds to the first half-byte (nibble) of an opcode. For example, any opcode that starts with ‘8’ (like 8xy0, 8xy1, etc.) would be found at table[8].
• table0: This is a sub-table for opcodes that begin with ‘0’. It helps differentiate between instructions like 00E0 and 00EE.
• table8: A sub-table for opcodes that start with ‘8’. As the CHIP-8 has multiple instructions starting with ‘8’, this table distinguishes between them based on the last nibble.
• tableE: Similarly, this table is for opcodes that begin with ‘E’, allowing for differentiation between instructions like ExA1 and Ex9E.
• tableF: This table handles a range of opcodes that start with ‘F’, with their differentiation based on the second byte of the opcode.

The add_table (shown below) function initializes these tables by associating each opcode with its respective function (or method). For instance, the opcode 8xy1 (OR operation between Vx and Vy) would be assigned to the Chip8::OP_8xy1 function.

Class Members

pub struct Chip8 {
    registers: [u8; 16],
    memory: [u8; 4096],
    index_register: u16,
    program_counter: u16,
    stack: [u16; 16],
    stack_pointer: u8,
    delay_timer: u8,
    sound_timer: u8,
    keypad: [u8; 16],
    pub video: [u32; 64 * 32],
    op_code: u16,
    table: [fn(&mut Chip8); 0xF+1],
    table0: [fn(&mut Chip8); 0xE+1],
    table8: [fn(&mut Chip8); 0xE+1],
    tableE: [fn(&mut Chip8); 0xE+1],
    tableF: [fn(&mut Chip8); 0x65+1],
    debug_mode: bool,
}

Loading a ROM

In a CHIP-8 emulator, the most crucial task after initializing the system is loading a ROM (Read Only Memory) that contains the game or program we want to run. Let’s delve into how the load_rom function works.

/// Loads a given rom into memory, starting from memory address 0x200
pub fn load_rom(&mut self, path: PathBuf) {

Purpose: This function is designed to load a provided ROM file into the emulator’s memory. It expects a PathBuf which is the path to the ROM file you want to load.

let start_address = 0x200;

Start Address: In the CHIP-8 system, the memory address 0x200 is the conventional starting point for most ROMs. The memory before this address is typically reserved for the system’s use, like the CHIP-8 interpreter itself and the font set.

let mut buf: Vec<u8> = Vec::new();

Byte Buffer: The buf vector will temporarily store the bytes read from the ROM file. It’s initialized as an empty vector of bytes (u8).

for byte in file.bytes() {
    let byte = match byte {
        Ok(byte) => byte,
        Err(error) => panic!("Provided rom is not a valid binary. {:?}", error),
    };
    buf.push(byte);
}

Reading the ROM: I then iterate over each byte in the ROM file. If a byte is successfully read, it’s added to the buf vector. However, if there’s any error while reading, the program will panic and provide an error message indicating that the ROM might not be a valid binary.

for i in 0..buf.len() {
    self.memory[start_address + i] = buf[i];
}

Loading into CHIP-8 Memory: Once the ROM’s bytes are stored in the buffer, I proceed to load each byte into the CHIP-8’s memory, starting from the start_address (0x200). This loop efficiently transfers the program from our buffer to the emulator’s memory.

if (self.debug_mode) {
    eprintln!("ROM Loaded");
}

Completion Message: Finally, if the debug_mode is enabled, a message indicating the successful loading of the ROM will be printed.

In conclusion, the load_rom function is essential for the emulator’s operation, ensuring that the game or program ROM is correctly loaded into memory and ready for execution.

Printing the Video Buffer

pub fn pretty_print_video(&mut self) {

Purpose: This function’s primary role is to render the video buffer of the CHIP-8 emulator in a more visually-pleasing manner on the console.

print!("\x1B[2J\x1B[1;1H");

Console Clear and Reset: This line employs ANSI escape codes to clear the console and move the cursor to the top-left position. This is done so that each frame from the video buffer is drawn on a fresh screen, preventing overlap and ensuring clarity.

let mut new_vec = self.video.iter().peekable();
let mut rows: Vec<Vec<_>> = vec![];

Initialization: We first convert the video buffer into an iterator that also supports the peek method. We also create an empty rows vector that will later store the video data in row-wise chunks.

while new_vec.peek().is_some() {
    let chunk: Vec<_> = new_vec.by_ref().take(64).collect();
    rows.push(chunk);
}

Chunking Video Buffer: CHIP-8 has a resolution of 64x32 pixels. Here, we take the linear video buffer and split it into rows of 64 pixels each. These rows are stored in the rows vector, making it easier to print them line by line.

for row in rows {
    let mut current_row = "".to_string();

Iterating Over Rows: For each row in our rows vector, we initiate an empty string current_row that will represent the current line of pixels.

for pixel in row {
    if (pixel == &(0xFFFFFFFF as u32)) {
        current_row.push('█');
    } else {
        current_row.push(' ');
    }
}

Pixel Representation: For each pixel in the row, we check its value. If the pixel’s value is 0xFFFFFFFF (which typically represents a white or “on” pixel in CHIP-8), we append a ‘█’ character to the current_row string. This character provides a filled visual representation. If the pixel is not ‘on’, we append a space to represent the “off” state.

println!("{}", current_row);

Printing the Row: Lastly, we print out the constructed current_row string. This process is repeated for all rows, thus rendering the entire frame from the video buffer.

Loading the Characters

fn load_fonts(&mut self) {

Purpose: This function is designed to load the CHIP-8’s built-in fontset into the emulator’s memory. This fontset contains graphical representations for the hexadecimal numbers 0 through F.

let fontset: [u8; 80] = [
            0xF0, 0x90, 0x90, 0x90, 0xF0, // 0
            0x20, 0x60, 0x20, 0x20, 0x70, // 1
            0xF0, 0x10, 0xF0, 0x80, 0xF0, // 2
            0xF0, 0x10, 0xF0, 0x10, 0xF0, // 3
            0x90, 0x90, 0xF0, 0x10, 0x10, // 4
            0xF0, 0x80, 0xF0, 0x10, 0xF0, // 5
            0xF0, 0x80, 0xF0, 0x90, 0xF0, // 6
            0xF0, 0x10, 0x20, 0x40, 0x40, // 7
            0xF0, 0x90, 0xF0, 0x90, 0xF0, // 8
            0xF0, 0x90, 0xF0, 0x10, 0xF0, // 9
            0xF0, 0x90, 0xF0, 0x90, 0x90, // A
            0xE0, 0x90, 0xE0, 0x90, 0xE0, // B
            0xF0, 0x80, 0x80, 0x80, 0xF0, // C
            0xE0, 0x90, 0x90, 0x90, 0xE0, // D
            0xF0, 0x80, 0xF0, 0x80, 0xF0, // E
            0xF0, 0x80, 0xF0, 0x80, 0x80, // F];
        ];

Fontset Declaration: The fontset array contains the binary representations of the 16 characters (0-F) used in CHIP-8. Each character is 5 bytes long and can be rendered on a 4x5 pixel grid.

let fontset_start_address = 0x50;

Fontset Memory Location: The fontset is loaded into a predefined starting address in memory, which is 0x50 or 80 in decimal. This is the standard location for the fontset in a typical CHIP-8 implementation.

if (self.debug_mode) {
    eprintln!("Loading fontset");
}

Debug Mode Check: If the emulator is running in debug mode, it will print a message to the console indicating that the fontset loading process has begun.

for i in 0..fontset.len() {
    self.memory[fontset_start_address + i as usize] = fontset[i as usize];
}

Loading the Fontset into Memory: This loop iterates through every byte in the fontset array and places each byte into the corresponding location in the emulator’s memory. It starts at fontset_start_address and continues until the entire fontset is loaded.

The Instructions

OP_00E0 - Clear Screen This opcode clears the screen by setting all pixels in the video buffer to zero.

    pub fn OP_00E0(&mut self) {
        // set video buffer to zero
        self.video = [0; 2048];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_00EE - Return from subroutine This opcode handles the end of a subroutine call. It decrements the stack pointer and sets the program counter back to the address at the top of the stack, effectively returning to where the subroutine was called.

    fn OP_00EE(&mut self) {
        self.stack_pointer = self.stack_pointer - 1;

        self.program_counter = self.stack[self.stack_pointer as usize];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_1nnn - Jump to location NNN This opcode sets the program counter directly to the address NNN, effectively making the program jump to this new address.

    fn OP_1nnn(&mut self) {
        // using 0x0FFF I can take the NNN from the opcode while leaving the one
        let address: u16 = self.op_code & 0x0FFF;

        self.program_counter = address;
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_2nnn - Call subroutine at location NNN This opcode calls a subroutine. The current program counter is saved to the stack, then the stack pointer is incremented, and the program counter is set to the address NNN.

    fn OP_2nnn(&mut self) {
        let address: u16 = self.op_code & 0x0FFF;

        self.stack[self.stack_pointer as usize] = self.program_counter;
        self.stack_pointer = self.stack_pointer + 1;
        self.program_counter = address;
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_3xkk - Skip next instruction if Vx = kk This opcode checks if register Vx contains the value kk. If true, it skips the next instruction by incrementing the program counter by 2.

    fn OP_3xkk(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let byte: u16 = self.op_code & 0x00FF;

        let byte: u8 = match u8::try_from(byte) {
            Ok(number) => number,
            Err(error) => panic!(
                "Could not turn u16 into u8 in OPCODE: 3XKK. Error: {}",
                error
            ),
        };

        if self.registers[Vx as usize] == byte {
            self.program_counter += 2;
        }
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_4xkk - Skip next instruction if Vx != kk Similar to the opcode above, but this opcode checks if register Vx does not contain the value kk. If true, it skips the next instruction.

    fn OP_4xkk(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let byte: u16 = self.op_code & 0x00FF;

        let byte: u8 = match u8::try_from(byte) {
            Ok(number) => number,
            Err(error) => panic!(
                "Could not turn u16 into u8 in OPCODE: 3XKK. Error: {}",
                error
            ),
        };

        // this != is the only difference from the function above
        if self.registers[Vx as usize] != byte {
            self.program_counter += 2;
        }
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_5xy0 - Skip next instruction if Vx = Vy This opcode compares the values of two registers, Vx and Vy. If their values are equal, the next instruction is skipped.

    fn OP_5xy0(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let Vy: u16 = (self.op_code & 0x00F0).checked_shr(4).unwrap_or(0);

        if self.registers[Vx as usize] == self.registers[Vy as usize] {
            self.program_counter += 2;
        }
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_6xkk - Set Vx = kk This opcode directly sets the value of register Vx to kk.

    fn OP_6xkk(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let byte: u16 = self.op_code & 0x00FF;

        let byte: u8 = match u8::try_from(byte) {
            Ok(number) => number,
            Err(error) => panic!(
                "Could not turn u16 into u8 in OPCODE 6XKK. Error: {}",
                error
            ),
        };

        self.registers[Vx as usize] = byte;
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }

    }

OP_7xkk - Set Vx = Vx + kk This opcode adds the value kk to register Vx and then stores the result back into Vx. If there’s an overflow, it wraps around.

    fn OP_7xkk(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let byte: u8 = (self.op_code as u8) & 0x00FF;

        self.registers[Vx as usize] = (((self.registers[Vx as usize] as u16) + byte as u16) % 256) as u8;

        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_8xy0 - Set Vx = Vy This opcode copies the value from register Vy to register Vx.

    fn OP_8xy0(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let Vy: u16 = (self.op_code & 0x00F0).checked_shr(4).unwrap_or(0);

        self.registers[Vx as usize] = self.registers[Vy as usize];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_8xy1 - Set Vx = Vx OR Vy This opcode performs a bitwise OR operation between the values in registers Vx and Vy, then stores the result back into Vx.

    fn OP_8xy1(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let Vy: u16 = (self.op_code & 0x00F0).checked_shr(4).unwrap_or(0);

        self.registers[Vx as usize] |= self.registers[Vy as usize];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_8xy2 - Set Vx = Vx AND Vy

Functionality: Bitwise AND of Vx and Vy.

Implementation Details: The values of registers Vx and Vy are obtained from the opcode. The value at register Vx is then bitwise AND-ed with the value at register Vy, and the result is stored back in Vx.

    fn OP_8xy2(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let Vy: u16 = (self.op_code & 0x00F0).checked_shr(4).unwrap_or(0);

        self.registers[Vx as usize] &= self.registers[Vy as usize];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }

OP_8xy3 - Set Vx = Vx XOR Vy

Functionality: Bitwise XOR of Vx and Vy.

Implementation Details: The values of registers Vx and Vy are obtained from the opcode. The value at register Vx is then bitwise XOR-ed with the value at register Vy, and the result is stored back in Vx.

    fn OP_8xy3(&mut self) {
        let Vx: u16 = (self.op_code & 0x0F00).checked_shr(8).unwrap_or(0);
        let Vy: u16 = (self.op_code & 0x00F0).checked_shr(4).unwrap_or(0);

        self.registers[Vx as usize] ^= self.registers[Vy as usize];
        if self.debug_mode {
            eprintln!("Ran opcode: {}", function_name!());
        }
    }