ProjectModel.h
7.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#ifndef PROJECTMODEL_H
#define PROJECTMODEL_H
#include <QAbstractItemModel>
#include <QSize>
#include <QMap>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <armadillo>
#include "Series.h"
#include "Segment.h"
#include "Episode.h"
#include "VideoFrame.h"
#include "MovieAnalyzer.h"
class ProjectModel: public QAbstractItemModel
{
Q_OBJECT
public:
ProjectModel(QObject *parent = 0);
~ProjectModel();
bool save(const QString &fName);
bool load(const QString &fName);
QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const;
QModelIndex parent(const QModelIndex &child = QModelIndex()) const;
QVariant data(const QModelIndex &index, int role) const;
int rowCount(const QModelIndex &parent = QModelIndex()) const;
int columnCount(const QModelIndex &parent = QModelIndex()) const;
QVariant headerData(int section, Qt::Orientation orientation, int role) const;
Qt::ItemFlags flags(const QModelIndex &index) const;
QModelIndex indexFromSegment(Segment *segment) const;
int getDepth() const;
QModelIndex getShotParentIndex() const;
void setModel(const QString &name, const QString &seriesName, int seasNbr, int epNbr, const QString &epName, const QString &epFName);
bool appendModel(int seasNbr, int epNbr, const QString &epName, const QString &epFName);
bool insertSubtitles(const QString &subFName);
bool speakerDiarization(const QString &subFName, VideoFrame::SpeakerSource source);
bool localSpkDiar(bool baseline, UtteranceTree::DistType dist, bool norm, UtteranceTree::AgrCrit agr, UtteranceTree::PartMeth partMeth, bool weight, bool sigma);
bool globalSpkDiar();
void extractShots(QString fName, int histoType, int nVBins, int nHBins, int nSBins, int metrics, qreal threshold1, qreal threshold2, int nVBlock, int nHBlock, bool iterate);
void labelSimilarShots(QString fName, int histoType, int nVBins, int nHBins, int nSBins, int metrics, qreal maxDist, int windowSize, int nVBlock, int nHBlock, bool iterate);
qreal evaluateShotDetection(bool displayResults, qreal thresh1, qreal thresh2) const;
qreal evaluateSimShotDetection(bool displayResults, qreal thresh1, qreal thresh2) const;
int retrieveShotPrevPositions(qint64 position, QList<qint64> &shotPositions);
void reset();
QString getName() const;
QString getBaseName() const;
QString getSeriesName() const;
void setSpkDiar(const QString &epFName);
public slots:
void setResolution(const QSize &resolution);
void setFps(qreal fps);
void appendVideoFrame(int id, qint64 position);
void initShotLevel(Segment *segment);
void insertShot(qint64 position, Segment::Source);
void insertShot(Segment *segment, Segment::Source);
void removeShot(Segment *segment, Segment::Source source);
void processSegmentation(bool checked, bool annot);
void retrieveSpeakers(bool checked);
void labelSimShot(qint64 position, int nCamera, Segment::Source source);
void setSpeaker(qint64 start, qint64 end, const QString &speaker, VideoFrame::SpeakerSource source);
void resetSpeaker(qint64 prevStart, qint64 prevEnd, qint64 start, qint64 end, bool resetSub, VideoFrame::SpeakerSource source);
void exportSubtitles(const QString &fName);
void improveSpkDiar();
void retrieveShotSub(qint64 position);
void playSubtitle(QList<int> utter);
void playSeg(QList<QPair<qint64, qint64>> segments);
void currentSub(qint64 position);
void extractIVectors(bool ubm, const QString &epFName);
void extractSpkIVectors(const QString &epFName, bool refSpk);
signals:
void modelChanged();
void positionChanged(qint64 position);
void truePositive(qint64 position);
void falsePositive(qint64 position);
void getShot(Segment *shot) const;
void segmentationRetrieved();
void getSpokenFrame(qint64 position, const QString &sub, const QString &speaker) const;
void viewSegmentation(bool checked, bool annot);
void speakersRetrieved(QList<QString> speakers);
void getCurrentPattern(const QList<QPair<int, qreal>> &subFeatures);
void getPatternFirstShot(const QList<QPair<int, qreal>> &subFeatures);
void getPatternSecondShot(const QList<QPair<int, qreal>> &subFeatures);
void playSegments(QList<QPair<qint64, qint64>> utterances);
void currentSubtitle(int subIdx);
void setDiarData(const arma::mat &E, const arma::mat &Sigma, const arma::mat &W);
void setDiarData(const arma::mat &E, const arma::mat &Sigma, const arma::mat &W, QMap<QString, QList<QPair<qreal, qreal>>> speakers);
private:
void eraseSubtitles(Segment *segment);
void clearSpeaker(Segment *segment, VideoFrame::SpeakerSource source);
bool shotLevelCreated(Segment *segment);
Segment * getFirstVideoFrame(Segment *segment);
qreal computePrecision(int tp, int fp) const;
qreal computeRecall(int tp, int fn) const;
qreal computeFScore(qreal precision, qreal recall) const;
qreal computeAccuracy(int tp, int fp, int fn, int tn) const;
void resetShotsToManual(Segment *segment, QList<Segment *> &toRemove);
void resetAutoCameraLabels(Segment *segment);
void depthFirstToShots(Segment *segment) const;
void depthFirstToSpokenFrames(Segment *segment, VideoFrame::SpeakerSource source) const;
void retrieveShotPositions(Segment *segment, QList<qint64> &shotPositions) const;
void retrieveSimCamLabels(Segment *segment, QList<int> &autCamLabels, QList<int> &manCamLabels) const;
void nFramesByCamLabel(Segment *segment, QMap<QString, QMap<QString, int>> &speakerList, Segment::Source vSource, VideoFrame::SpeakerSource sSource) const;
void retrieveSpokenVFrames(Segment *segment, QList<VideoFrame *> &spkVFrames, VideoFrame::SpeakerSource source) const;
void retrieveVFrames(Segment *segment, QList<VideoFrame *> &vFrames) const;
void retrieveSpeakersList(Segment *segment, QList<QString> &speakers, VideoFrame::SpeakerSource source) const;
void retrieveShotUtterances();
void retrieveShotPatterns();
void retrieveShotLabels(Segment *segment, QList<QString> &shotLabels) const;
void retrieveSubPositionsLabels(QList<qint64> &subStarts, QList<qint64> &subEnds, QList<QString> &subRefLbl) const;
bool testShotPattern(const QList<QString> &lblWindow, int pattSize);
QPair<qint64, qint64> adjustSubBoundaries(qint64 subStart, qint64 subEnd, QList<QPair<qint64, qint64>> shotPattBound);
QString normalizedPattern(const QString &firstLabel, const QString &secondLabel);
QString mergePatterns(QString firstLabel, QString secondLabel, QList<QPair<int, qreal>> list1, QList<QPair<int, qreal>> list2, QList<QPair<int, qreal>> &mergedList);
QString completePatternLabel(const QString &label, QMap<QString, QList<QPair<int, qreal>>> shotPatterns);
bool interPatterns(QString firstLabel, QString secondLabel);
QStringList appendStringList(const QStringList &list1, const QStringList &list2);
void mergePatternBoundaries(QMap<QString, QList<QPair<qint64, qint64>>> &shotPattBound);
void exportGlobSpkRef(const QString &fName);
void exportLocSpkRef(const QString &fName);
arma::mat genWMat(QMap<QString, QList<int>> spkIdx, const arma::mat &X);
QString m_name;
QString m_baseName;
Series *m_series;
Episode *m_episode;
MovieAnalyzer *m_movieAnalyzer;
QList<QPair<qint64, qint64>> m_subBound;
QMap<QString, QList<QPair<int, qreal>>> m_shotUtterances;
QMap<QString, QList<QPair<int, qreal>>> m_shotPatterns;
QMap<QString, QList<QPair<qint64, qint64>>> m_shotPattBound;
QMap<QString, QList<QPair<qint64, qint64>>> m_strictShotPattBound;
QList<QString> m_subRefLbl;
cv::VideoCapture m_cap;
arma::mat m_W;
};
#endif