sixel: improve the renderer (#143)

In the current implementation, when text is written over an image, we
have to cut the entire text line out of the image, regardless of how
long the text is. It doesn't look good, but it was a design choice for
the following reasons:
1) To keep the sixel engine as fast as possible
2) Most applications do not write text on the images anyway

To bring the st terminal in line with other terminals that support
sixels, I have now improved the sixel renderer so that the images can
now have gaps, which allows the text to be printed inside the images.
The changes should not affect performance in normal cases. Only when the
renderer has to deal with the text there might be some performance hits
depending on how many gaps there are in the images.

patch/reflow.c

@@ -77,7 +77,7 @@ treflow_moveimages(int oldy, int newy)
 void
 treflow(int col, int row)
 {
-	int i, j, x, x2;
+	int i, j;
 	int oce, nce, bot, scr;
 	int ox = 0, oy = -term.histf, nx = 0, ny = -1, len;
 	int cy = -1; /* proxy for new y coordinate of cursor */
@@ -220,13 +220,12 @@ treflow(int col, int row)
 	}
 
 	/* expand images into new text cells */
-	for (im = term.images; im; im = next) {
+	for (im = term.images; im; im = im->next) {
-		next = im->next;
+		j = MIN(im->x + im->cols, col);
-		if (im->x < col) {
+		line = TLINE(im->y);
-			line = TLINE(im->y);
+		for (i = im->x; i < j; i++) {
-			x2 = MIN(im->x + im->cols, col);
+			if (!(line[i].mode & ATTR_SET))
-			for (x = im->x; x < x2; x++)
+				line[i].mode |= ATTR_SIXEL;
-				line[x].mode |= ATTR_SIXEL;
 		}
 	}
 	#endif // SIXEL_PATCH

st.c

@@ -2748,7 +2748,11 @@ strhandle(void)
 			} else {
 				term.images = newimages;
 			}
+			#if COLUMNS_PATCH && !REFLOW
+			x2 = MIN(x2, term.maxcol) - 1;
+			#else
 			x2 = MIN(x2, term.col) - 1;
+			#endif // COLUMNS_PATCH
 			if (IS_SET(MODE_SIXEL_SDM)) {
 				/* Sixel display mode: put the sixel in the upper left corner of
 				 * the screen, disable scrolling (the sixel will be truncated if
@@ -3521,7 +3525,7 @@ tresize(int col, int row)
 	#endif // COLUMNS_PATCH
 	int *bp;
 	#if SIXEL_PATCH
-	int x, x2;
+	int x2;
 	Line line;
 	ImageList *im, *next;
 	#endif // SIXEL_PATCH
@@ -3628,8 +3632,7 @@ tresize(int col, int row)
 	}
 
 	#if SIXEL_PATCH
-	/* expand images into new text cells to prevent them from being deleted in
+	/* expand images into new text cells */
-	 * xfinishdraw() that draws the images */
 	for (i = 0; i < 2; i++) {
 		for (im = term.images; im; im = next) {
 			next = im->next;
@@ -3654,9 +3657,9 @@ tresize(int col, int row)
 			}
 			line = term.line[im->y];
 			#endif // SCROLLBACK_PATCH
-			x2 = MIN(im->x + im->cols, term.col);
+			x2 = MIN(im->x + im->cols, col) - 1;
-			for (x = im->x; x < x2; x++)
+			if (mincol < col && x2 >= mincol && im->x < col)
-				line[x].mode |= ATTR_SIXEL;
+				tsetsixelattr(line, MAX(im->x, mincol), x2);
 		}
 		tswapscreen();
 	}

x.c

@@ -3167,9 +3167,14 @@ xfinishdraw(void)
 	ImageList *im, *next;
 	Imlib_Image origin, scaled;
 	XGCValues gcvalues;
-	GC gc;
+	GC gc = NULL;
 	int width, height;
-	int x, x2, del, destx, desty;
+	int del, desty, mode, x1, x2, xend;
+	#if ANYSIZE_PATCH
+	int bw = win.hborderpx, bh = win.vborderpx;
+	#else
+	int bw = borderpx, bh = borderpx;
+	#endif // ANYSIZE_PATCH
 	Line line;
 	#endif // SIXEL_PATCH
 
@@ -3260,43 +3265,50 @@ xfinishdraw(void)
 			}
 		}
 
-		/* clip the image so it does not go over to borders */
+		/* create GC */
-		x2 = MIN(im->x + im->cols, term.col);
+		if (!gc) {
-		width = MIN(width, (x2 - im->x) * win.cw);
+			memset(&gcvalues, 0, sizeof(gcvalues));
-
+			gcvalues.graphics_exposures = False;
-		/* delete the image if the text cells behind it have been changed */
+			gc = XCreateGC(xw.dpy, xw.win, GCGraphicsExposures, &gcvalues);
-		#if SCROLLBACK_PATCH || REFLOW_PATCH
-		line = TLINE(im->y);
-		#else
-		line = term.line[im->y];
-		#endif // SCROLLBACK_PATCH | REFLOW_PATCH
-		for (del = 0, x = im->x; x < x2; x++) {
-			if ((del = !(line[x].mode & ATTR_SIXEL)))
-				break;
-		}
-		if (del) {
-			delete_image(im);
-			continue;
 		}
 
-		/* draw the image */
+		/* set the clip mask */
-		memset(&gcvalues, 0, sizeof(gcvalues));
+		desty = bh + im->y * win.ch;
-		gcvalues.graphics_exposures = False;
-		gc = XCreateGC(xw.dpy, xw.win, GCGraphicsExposures, &gcvalues);
-		#if ANYSIZE_PATCH
-		destx = win.hborderpx + im->x * win.cw;
-		desty = win.vborderpx + im->y * win.ch;
-		#else
-		destx = borderpx + im->x * win.cw;
-		desty = borderpx + im->y * win.ch;
-		#endif // ANYSIZE_PATCH
 		if (im->clipmask) {
 			XSetClipMask(xw.dpy, gc, (Drawable)im->clipmask);
-			XSetClipOrigin(xw.dpy, gc, destx, desty);
+			XSetClipOrigin(xw.dpy, gc, bw + im->x * win.cw, desty);
 		}
-		XCopyArea(xw.dpy, (Drawable)im->pixmap, xw.buf, gc, 0, 0, width, height, destx, desty);
+
-		XFreeGC(xw.dpy, gc);
+		/* draw only the parts of the image that are not erased */
+		#if SCROLLBACK_PATCH || REFLOW_PATCH
+		line = TLINE(im->y) + im->x;
+		#else
+		line = term.line[im->y] + im->x;
+		#endif // SCROLLBACK_PATCH || REFLOW_PATCH
+		xend = MIN(im->x + im->cols, term.col);
+		for (del = 1, x1 = im->x; x1 < xend; x1 = x2) {
+			mode = line->mode & ATTR_SIXEL;
+			for (x2 = x1 + 1; x2 < xend; x2++) {
+				if (((++line)->mode & ATTR_SIXEL) != mode)
+					break;
+			}
+			if (mode) {
+				XCopyArea(xw.dpy, (Drawable)im->pixmap, xw.buf, gc,
+				    (x1 - im->x) * win.cw, 0,
+				    MIN((x2 - x1) * win.cw, width - (x1 - im->x) * win.cw), height,
+				    bw + x1 * win.cw, desty);
+				del = 0;
+			}
+		}
+		if (im->clipmask)
+			XSetClipMask(xw.dpy, gc, None);
+
+		/* if all the parts are erased, we can delete the entire image */
+		if (del && im->x + im->cols <= term.col)
+			delete_image(im);
 	}
+	if (gc)
+		XFreeGC(xw.dpy, gc);
 	#endif // SIXEL_PATCH
 
 	#if !SINGLE_DRAWABLE_BUFFER_PATCH