LWJGL3 Visual artifacts caused by different projections - glsl

Question

I have a very simple glsl vertex shader, which is the following:
#version 430
in vec2 vertices;
in vec2 textures;
out vec2 tex_coords;
uniform mat4 projection;
uniform mat4 texModifier;
void main() {
tex_coords = (texModifier * vec4(textures,0,1)).xy;
gl_Position = projection * vec4(vertices, 0, 1);
}
This shader is used to render text. The text is just a font sheet where I change the texModifier so that the mat4 coordinates are over the specific letter I want to draw. This works fine, but I noticed that different projection matrixes cause different visual artifacts.
Here is the an example:
This is the text using its original position.
And this is the the same text only the projection matrix is translated 0.1f / 1920 to the left. The visual artifacts change from being around commas and r to being around the u, S and more. These artifacts appear even when using linear interpolation.
This is the function used to draw text:
public void drawText(Shader shader, CharSequence text, float x, float y, int letters) {
if(letters == -1)
letters = text.length();
Model model;
Matrix4f projection = new Matrix4f();
Texture fontSheet = texture;
int textHeight = getTextHeight(text);
projection.setOrtho(-1, 1, -1, 1, -1, 1);
float drawX = x;
float drawY = y;
if (textHeight > fontHeight) {
drawY += textHeight - fontHeight;
}
shader.bind();
shader.setUniform("sampler", 0);
fontSheet.bind(0);
for (int i = 0; i < text.length() && i < letters; i++) {
char ch = text.charAt(i);
if (ch == '\n') {
/* Line feed, set x and y to draw at the next line */
drawY -= fontHeight;
drawX = x;
continue;
}
if (ch == '\r') {
/* Carriage return, just skip it */
continue;
}
Glyph g = glyphs.get(ch);
model = Model.createQuad((float) g.width / 2, (float) g.height / 2);
projection.setTranslation(((float) g.width / 2 + drawX) / Main.WINDOW_WIDTH, drawY / Main.WINDOW_HEIGHT, 0);
Matrix4f texModifier =
new Matrix4f()
.translate((float) g.x / fontSheet.getWidth(), 0, 0)
.scale((float) g.width / fontSheet.getWidth(), -(float) g.height / fontSheet.getHeight(), 1);
shader.setUniform("projection", projection);
shader.setUniform("texModifier", texModifier);
model.render();
drawX += g.width;
}
}
Why is the projection causing theses artifacts?