Strange behavior of QSyntaxHighlighter in Qt3

[lorebett]

The Qt3 implementation of QSyntaxHighlighter seems to be a little bit buggy or at least the semantics of highlightParagraph, in particular, the semantics of the states of the paragraph is not completely clear.

As an example consider the code below, which highlights some keywords (in blue) and handles two types of comments: standard /* */ comments (in red) and /** */ doxyfile comments in
green; for simplicity we assume that comment delimiters appear on a single line.

#include <qapplication.h>
#include <qsyntaxhighlighter.h>
#include <qtextedit.h>
#include <iostream>
 
#define NORMAL_STATE 0
#define COMMENT_STATE 1
#define DOXY_STATE 2
 
class SyntaxHighlighter: public QSyntaxHighlighter {
    QStringList keywordList;
    QColor keyColor;
    QFont keyFont;
    QColor commentColor;
    QFont commentFont;
    QColor commentDoxyColor;
 
public:
    SyntaxHighlighter(QTextEdit* textEdit) :
        QSyntaxHighlighter(textEdit) {
        keyFont.setBold(true);
        keyFont.setItalic(true);
        keyFont.setUnderline(true);
        keyColor = Qt::blue;
 
        commentColor = Qt::red;
        commentDoxyColor = Qt::green;
 
        commentFont.setItalic(true);
    }
 
    ~SyntaxHighlighter() {
    }
 
    void addKeyword(const QString &str) {
        keywordList << str;
    }
 
    int highlightParagraph(const QString & text, int endStateOfLastPara) {
        int index = 0;
        int count = 0;
        QStringList::iterator it = keywordList.begin();
 
        std::cout << "prev par state: " << endStateOfLastPara << std::endl;
        std::cout << "text: " << text << std::endl;
 
        // Assume the text to not contain keywords
        setFormat(0, text.length(), textEdit()->font(), Qt::black);
 
        int currentState = endStateOfLastPara;
 
        if (endStateOfLastPara == NORMAL_STATE || endStateOfLastPara < 0) {
            // keep it simple and assume that comment delimiters appear on a single line
            if (text.contains("/**")) {
                setFormat(0, text.length(), commentFont, commentDoxyColor);
                currentState = DOXY_STATE;
            } else if (text.contains("/*")) {
                setFormat(0, text.length(), commentFont, commentColor);
                currentState = COMMENT_STATE;
            } else {
                // Look for keywords
                while (it != keywordList.end()) {
                    count = text.contains(*it);
                    if (count) {
                        for (int i = 0; i < count; i++) {
                            index = text.find(*it);
                            setFormat(index, (*it).length(), keyFont, keyColor);
                            ++index;
                        }
                    }
                    ++it;
                }
            }
        } else if (endStateOfLastPara == COMMENT_STATE) {
            setFormat(0, text.length(), commentFont, commentColor);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        } else if (endStateOfLastPara == DOXY_STATE) {
            setFormat(0, text.length(), commentFont, commentDoxyColor);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        }
 
        std::cout << "current state: " << currentState << "\n" << std::endl;
 
        return currentState;
    }
};
 
include <qapplication.h>
#include <qtextedit.h>
 
int main(int argc, char **argv)
{
    QApplication a(argc, argv);
 
    QTextEdit te;
    SyntaxHighlighter* sh = new SyntaxHighlighter(&te);
    sh->addKeyword("class");
    sh->addKeyword("if");
    sh->addKeyword("while");
    sh->addKeyword("for");
 
    te.setText("/*\nfoo\n*/\nclass\nwhile");
    te.show();
    a.setMainWidget(&te);
 
    return a.exec();
}

#include <qapplication.h>
#include <qsyntaxhighlighter.h>
#include <qtextedit.h>
#include <iostream>

#define NORMAL_STATE 0
#define COMMENT_STATE 1
#define DOXY_STATE 2

class SyntaxHighlighter: public QSyntaxHighlighter {
    QStringList keywordList;
    QColor keyColor;
    QFont keyFont;
    QColor commentColor;
    QFont commentFont;
    QColor commentDoxyColor;

public:
    SyntaxHighlighter(QTextEdit* textEdit) :
        QSyntaxHighlighter(textEdit) {
        keyFont.setBold(true);
        keyFont.setItalic(true);
        keyFont.setUnderline(true);
        keyColor = Qt::blue;

        commentColor = Qt::red;
        commentDoxyColor = Qt::green;

        commentFont.setItalic(true);
    }

    ~SyntaxHighlighter() {
    }

    void addKeyword(const QString &str) {
        keywordList << str;
    }

    int highlightParagraph(const QString & text, int endStateOfLastPara) {
        int index = 0;
        int count = 0;
        QStringList::iterator it = keywordList.begin();

        std::cout << "prev par state: " << endStateOfLastPara << std::endl;
        std::cout << "text: " << text << std::endl;

        // Assume the text to not contain keywords
        setFormat(0, text.length(), textEdit()->font(), Qt::black);

        int currentState = endStateOfLastPara;

        if (endStateOfLastPara == NORMAL_STATE || endStateOfLastPara < 0) {
            // keep it simple and assume that comment delimiters appear on a single line
            if (text.contains("/**")) {
                setFormat(0, text.length(), commentFont, commentDoxyColor);
                currentState = DOXY_STATE;
            } else if (text.contains("/*")) {
                setFormat(0, text.length(), commentFont, commentColor);
                currentState = COMMENT_STATE;
            } else {
                // Look for keywords
                while (it != keywordList.end()) {
                    count = text.contains(*it);
                    if (count) {
                        for (int i = 0; i < count; i++) {
                            index = text.find(*it);
                            setFormat(index, (*it).length(), keyFont, keyColor);
                            ++index;
                        }
                    }
                    ++it;
                }
            }
        } else if (endStateOfLastPara == COMMENT_STATE) {
            setFormat(0, text.length(), commentFont, commentColor);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        } else if (endStateOfLastPara == DOXY_STATE) {
            setFormat(0, text.length(), commentFont, commentDoxyColor);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        }

        std::cout << "current state: " << currentState << "\n" << std::endl;

        return currentState;
    }
};

include <qapplication.h>
#include <qtextedit.h>

int main(int argc, char **argv)
{
    QApplication a(argc, argv);

    QTextEdit te;
    SyntaxHighlighter* sh = new SyntaxHighlighter(&te);
    sh->addKeyword("class");
    sh->addKeyword("if");
    sh->addKeyword("while");
    sh->addKeyword("for");

    te.setText("/*\nfoo\n*/\nclass\nwhile");
    te.show();
    a.setMainWidget(&te);

    return a.exec();
}

To copy to clipboard, switch view to plain text mode 

The code contains some debug statements that are helpful:
- the state of the previous par (i.e. endStateOfLastPara)
- the text to highlight
- the state value that we're about to return

Assume this text for the contents of QTextEdit (automatically inserted in the main):

/*
foo
*/
class
while

The documentation of highlightParagraph, concerning the returned value says:

int QSyntaxHighlighter::highlightParagraph ( const QString & text, int endStateOfLastPara )

"The value you return is up to you. We recommend only returning 0
(to signify that this paragraph's syntax highlighting does not affect the following paragraph), or a positive integer (to signify that this paragraph has ended in the middle of a paragraph spanning construct)."

if you modify the line with "foo" (e.g., type a char), you'll see that the previous state is COMMENT_STATE, and that we return COMMENT_STATE (i.e., non 0), however, only this paragraph is highlighted, and this is correct, thus a non 0 returned value does not mean: highlight also the next paragraph; from the documentation above one might have the impression that this is what's
expected, as a negation of "returning 0 (to signify that this paragraph's syntax highlighting does not affect the following paragraph)".
However, the behavior is correct (there's no need to highlight the following paragraphs), since the state of the paragraph did not change.

Now, try to change "/*" into "/**", the comment is correctly highlighted in green (doxy comment), however, one would expect that only the parts containing the comments are highlighted, since nothing has changed for the last two lines.
Moreover, when highlighting "*/" we return 0 (NORMAL_STATE) "to signify that this paragraph's syntax highlighting does not
affect the following paragraph".

However, this does not happen: all the lines down to the end of the document are highlighted again (see the debug info). Thus, the semantics of the 0 seems not to be respected.

Note that in a medium size document, this causes a lot of overhead.

Indeed the semantics that is probably used is

1. if the state of a paragraph did not change (w.r.t. to its own previous state) then don't highlight the paragraphs that follow (and this perfectly makes sense)
2. if such state changes highlight everything else up to the end of the document

the second point is completely wrong, and as said above, causes a lot
of overhead.
In particular, the semantics "We recommend only returning 0
(to signify that this paragraph's syntax highlighting does not
affect the following paragraph)" should be always respected.

This is also made worse when you insert a new line within the comment environment (try to insert a new line after foo): again, from that line on, the whole document will be highlighted again. This is tragic.

Is this a known issue in Qt3?

thanks in advance

[lorebett]

Note that qt4 version of QSyntaxHighlighter does this right (relying on
getCurrentBlockState and setCurrentBlockState). For instance, if you try this code in Qt4 you'll see that only the parts that really need to be highlighted are processed:

#include <QApplication>
#include <QSyntaxHighlighter>
#include <QTextEdit>
#include <QTextCharFormat>
#include <iostream>
 
#define NORMAL_STATE 0
#define COMMENT_STATE 1
#define DOXY_STATE 2
 
class SyntaxHighlighter: public QSyntaxHighlighter {
    QStringList keywordList;
 
    QTextCharFormat keyFormat;
    QTextCharFormat commentFormat;
    QTextCharFormat commentDoxyFormat;
 
public:
    SyntaxHighlighter(QTextEdit* textEdit) :
        QSyntaxHighlighter(textEdit) {
        QColor keyColor;
        QFont keyFont;
        QColor commentColor;
        QFont commentFont;
        QColor commentDoxyColor;
        keyFont.setBold(true);
        keyFont.setItalic(true);
        keyFont.setUnderline(true);
        keyColor = Qt::blue;
 
        commentColor = Qt::red;
        commentDoxyColor = Qt::green;
 
        commentFont.setItalic(true);
 
        keyFormat.setFont(keyFont);
        keyFormat.setForeground(keyColor);
        commentFormat.setFont(commentFont);
        commentFormat.setForeground(commentColor);
        commentDoxyFormat.setFont(commentFont);
        commentDoxyFormat.setForeground(commentDoxyColor);
    }
 
    ~SyntaxHighlighter() {
    }
 
    void addKeyword(const QString &str) {
        keywordList << str;
    }
 
    void highlightBlock(const QString & text) {
        int index = 0;
        int count = 0;
        QStringList::iterator it = keywordList.begin();
 
        int endStateOfLastPara = previousBlockState();
        int currentState = endStateOfLastPara;
 
        std::cout << "prev par state: " << endStateOfLastPara << std::endl;
        std::cout << "text: " << text.toStdString() << std::endl;
 
        // Assume the text to not contain keywords
        setFormat(0, text.length(), Qt::black);
 
        if (endStateOfLastPara == NORMAL_STATE || endStateOfLastPara < 0) {
            // keep it simple and assume that comment delimiters appear on a single line
            if (text.contains("/**")) {
                setFormat(0, text.length(), commentDoxyFormat);
                currentState = DOXY_STATE;
            } else if (text.contains("/*")) {
                setFormat(0, text.length(), commentFormat);
                currentState = COMMENT_STATE;
            } else {
                // Look for keywords
                while (it != keywordList.end()) {
                    count = text.count(*it);
                    if (count) {
                        for (int i = 0; i < count; i++) {
                            index = text.indexOf(*it);
                            setFormat(index, (*it).length(), keyFormat);
                            ++index;
                        }
                    }
                    ++it;
                }
            }
        } else if (endStateOfLastPara == COMMENT_STATE) {
            setFormat(0, text.length(), commentFormat);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        } else if (endStateOfLastPara == DOXY_STATE) {
            setFormat(0, text.length(), commentDoxyFormat);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        }
 
        std::cout << "current state: " << currentState << "\n" << std::endl;
 
        setCurrentBlockState(currentState);
    }
};
 
#include <QApplication>
#include <QTextEdit>
#include <QMainWindow>
 
int main(int argc, char **argv)
{
    QApplication a(argc, argv);
    QMainWindow window;
    QTextEdit te;
    SyntaxHighlighter* sh = new SyntaxHighlighter(&te);
    sh->addKeyword("class");
    sh->addKeyword("if");
    sh->addKeyword("while");
    sh->addKeyword("for");
 
    window.setCentralWidget(&te);
    te.setText("/*\nfoo\n*/\nclass\nwhile");
    window.show();
 
    return a.exec();
}

#include <QApplication>
#include <QSyntaxHighlighter>
#include <QTextEdit>
#include <QTextCharFormat>
#include <iostream>

#define NORMAL_STATE 0
#define COMMENT_STATE 1
#define DOXY_STATE 2

class SyntaxHighlighter: public QSyntaxHighlighter {
    QStringList keywordList;

    QTextCharFormat keyFormat;
    QTextCharFormat commentFormat;
    QTextCharFormat commentDoxyFormat;

public:
    SyntaxHighlighter(QTextEdit* textEdit) :
        QSyntaxHighlighter(textEdit) {
        QColor keyColor;
        QFont keyFont;
        QColor commentColor;
        QFont commentFont;
        QColor commentDoxyColor;
        keyFont.setBold(true);
        keyFont.setItalic(true);
        keyFont.setUnderline(true);
        keyColor = Qt::blue;

        commentColor = Qt::red;
        commentDoxyColor = Qt::green;

        commentFont.setItalic(true);

        keyFormat.setFont(keyFont);
        keyFormat.setForeground(keyColor);
        commentFormat.setFont(commentFont);
        commentFormat.setForeground(commentColor);
        commentDoxyFormat.setFont(commentFont);
        commentDoxyFormat.setForeground(commentDoxyColor);
    }

    ~SyntaxHighlighter() {
    }

    void addKeyword(const QString &str) {
        keywordList << str;
    }

    void highlightBlock(const QString & text) {
        int index = 0;
        int count = 0;
        QStringList::iterator it = keywordList.begin();

        int endStateOfLastPara = previousBlockState();
        int currentState = endStateOfLastPara;

        std::cout << "prev par state: " << endStateOfLastPara << std::endl;
        std::cout << "text: " << text.toStdString() << std::endl;

        // Assume the text to not contain keywords
        setFormat(0, text.length(), Qt::black);

        if (endStateOfLastPara == NORMAL_STATE || endStateOfLastPara < 0) {
            // keep it simple and assume that comment delimiters appear on a single line
            if (text.contains("/**")) {
                setFormat(0, text.length(), commentDoxyFormat);
                currentState = DOXY_STATE;
            } else if (text.contains("/*")) {
                setFormat(0, text.length(), commentFormat);
                currentState = COMMENT_STATE;
            } else {
                // Look for keywords
                while (it != keywordList.end()) {
                    count = text.count(*it);
                    if (count) {
                        for (int i = 0; i < count; i++) {
                            index = text.indexOf(*it);
                            setFormat(index, (*it).length(), keyFormat);
                            ++index;
                        }
                    }
                    ++it;
                }
            }
        } else if (endStateOfLastPara == COMMENT_STATE) {
            setFormat(0, text.length(), commentFormat);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        } else if (endStateOfLastPara == DOXY_STATE) {
            setFormat(0, text.length(), commentDoxyFormat);
            if (text.contains("*/"))
                currentState = NORMAL_STATE;
        }

        std::cout << "current state: " << currentState << "\n" << std::endl;

        setCurrentBlockState(currentState);
    }
};

#include <QApplication>
#include <QTextEdit>
#include <QMainWindow>

int main(int argc, char **argv)
{
    QApplication a(argc, argv);
    QMainWindow window;
    QTextEdit te;
    SyntaxHighlighter* sh = new SyntaxHighlighter(&te);
    sh->addKeyword("class");
    sh->addKeyword("if");
    sh->addKeyword("while");
    sh->addKeyword("for");

    window.setCentralWidget(&te);
    te.setText("/*\nfoo\n*/\nclass\nwhile");
    window.show();

    return a.exec();
}

To copy to clipboard, switch view to plain text mode